00000100:00100000:3.0:1766097872.318730:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432829952:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (630us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097872.318746:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047
00000100:00100000:3.0:1766097872.318780:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830016
00000100:00100000:3.0:1766097872.318807:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830144
00000100:00100000:3.0:1766097872.318828:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1527
00000100:00100000:3.0:1766097872.318832:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432830016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097872.318867:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432830016:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (710us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097872.318873:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1527
00000100:00100000:3.0:1766097872.318885:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1528
00000100:00100000:3.0:1766097872.318888:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c51052e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432830144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097872.318914:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c51052e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432830144:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (512us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097872.318919:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1528
00000100:00100000:3.0:1766097872.318972:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090c2623e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432830144:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097872.318984:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000113a6a27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432830016:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097872.319016:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006394e12b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432830080:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766097872.319043:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097872.319070:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830080
00000100:00100000:3.0:1766097872.319088:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900
00000100:00100000:3.0:1766097872.319092:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000113a6a27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432830080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766097872.325664:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097872.325815:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000113a6a27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432830080:12345-0@lo:8:kworker/u8:3.0 Request processed in 6719us (6769us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097872.325852:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900
00000100:00100000:3.0:1766097876.770148:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417159872
00000100:00100000:3.0:1766097876.770195:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1529
00000100:00100000:3.0:1766097876.770200:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000812d02e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417159872:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097876.770241:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000812d02e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417159872:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 41us (1503us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097876.770248:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1529
00000100:00100000:3.0:1766097877.442742:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830272
00000100:00100000:3.0:1766097877.442783:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049
00000100:00100000:3.0:1766097877.442788:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432830272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097877.442832:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432830272:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (357us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097877.442840:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049
00000100:00100000:3.0:1766097877.442893:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432830400:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766097877.442924:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097877.442988:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036aa7ae3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432830272:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097877.443021:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830400
00000100:00100000:3.0:1766097877.443044:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 902
00000100:00100000:3.0:1766097877.443048:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432830400:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766097877.443142:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097877.443178:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432830400:12345-0@lo:8:kworker/u8:3.0 Request processed in 128us (253us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097877.443185:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 902
00000100:00080000:3.0:1766097877.443209:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097877.443224:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097877.443228:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432830400:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766097877.443256:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad932d0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432830464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097877.443286:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097877.443313:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830464
00000100:00100000:3.0:1766097877.443336:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1531
00000100:00100000:3.0:1766097877.443340:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432830464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097877.443371:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432830464:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (85us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097877.443378:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1531
00000100:00100000:3.0:1766097877.443405:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad932d0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432830464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097881.891829:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766097881.891893:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766097881.891934:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766097881.891984:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417160128
00000100:00100000:3.0:1766097881.892031:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 903
00000100:00100000:3.0:1766097881.892040:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000633e633e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417160128:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766097881.892149:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097881.892224:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000633e633e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417160128:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 184us (289us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097881.892232:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 903
00000100:00080000:3.0:1766097882.559147:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766097882.559157:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766097882.559161:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766097882.559189:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2385
00000100:00080000:3.0:1766097882.559192:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766097882.559199:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766097882.559223:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000113a6a27 x1851885432830720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766097882.559291:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060c730a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432830848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097882.559409:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830720
00000100:00100000:3.0:1766097882.559450:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 904
00000100:00100000:3.0:1766097882.559458:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432830720:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766097882.559595:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097882.559642:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432830720:12345-0@lo:8:kworker/u8:3.0 Request processed in 184us (264us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097882.559649:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 904
00000100:00100000:3.0:1766097882.559674:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1534
00000100:00100000:3.0:1766097882.559677:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006141d7f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432830656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097882.559702:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006141d7f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432830656:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (166us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097882.559707:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1534
00000100:00080000:3.0:1766097882.559733:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097882.559736:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097882.559739:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000113a6a27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432830720:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766097882.559754:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097882.559768:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766097882.559773:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432830592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097882.559800:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766097882.559829:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000060c730a6 x1851885432830848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097889 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766097882.559837:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060c730a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432830848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097882.559858:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830592
00000100:00100000:3.0:1766097882.559882:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1051
00000100:00100000:3.0:1766097882.559886:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432830592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097882.559930:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432830592:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (129us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097882.559937:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1051
00000100:00100000:3.0:1766097887.004424:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766097887.004538:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766097887.006651:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00080000:3.0:1766097887.678172:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766097887.678177:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766097887.678180:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766097887.678186:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2391
00000100:00080000:3.0:1766097887.678190:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766097887.678197:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766097887.678221:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006394e12b x1851885432831040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766097887.678306:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432830912
00000100:00100000:3.0:1766097887.678348:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053
00000100:00100000:3.0:1766097887.678354:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432830912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097887.678419:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432830912:12345-0@lo:400:kworker/u8:0.0 Request processed in 65us (222us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097887.678451:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053
00000100:00100000:3.0:1766097887.678506:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006394e12b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432831040:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097887.678544:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097887.679227:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432830912:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766097887.680223:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000065ef528 x1851885432831168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097894 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097887.680234:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000065ef528 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432831168:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766097887.680346:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097887.680351:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097887.680356:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006394e12b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432831040:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097892.127845:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417160512
00000100:00100000:3.0:1766097892.127881:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 907
00000100:00100000:3.0:1766097892.127885:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1fe4f18 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417160512:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766097892.127994:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097892.128209:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1fe4f18 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417160512:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 322us (760us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097892.128215:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 907
00000100:00100000:3.0:1766097892.800149:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3b84863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432831232:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097892.800502:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097892.800553:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432831296
00000100:00100000:3.0:1766097892.800588:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432831424
00000100:00100000:3.0:1766097892.800614:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1540
00000100:00100000:3.0:1766097892.800620:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432831424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097892.800679:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432831424:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (209us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097892.800686:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1540
00000100:00100000:3.0:1766097892.800721:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432831360
00000100:00100000:3.0:1766097892.800744:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908
00000100:00100000:3.0:1766097892.800751:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c4f798bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432831360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766097892.801743:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097892.801852:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c4f798bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432831360:12345-0@lo:8:kworker/u8:3.0 Request processed in 1100us (1289us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097892.801859:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908
00000100:00100000:3.0:1766097897.248267:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417160576
00000100:00100000:3.0:1766097897.248304:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1056
00000100:00100000:3.0:1766097897.248308:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417160576:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097897.248344:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417160576:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 35us (2674us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097897.248351:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1056
00000100:00100000:3.0:1766097897.919172:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c8141ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432831552:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097897.923307:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097897.923361:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432831552
00000100:00100000:3.0:1766097897.923394:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1057
00000100:00100000:3.0:1766097897.923399:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432831552:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097897.923441:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432831552:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (136us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097897.923448:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1057
00000100:00100000:3.0:1766097897.923510:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c8141ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432831552:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097902.366627:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417160768
00000100:00100000:3.0:1766097902.366668:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1058
00000100:00100000:3.0:1766097902.366677:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417160768:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097902.366720:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417160768:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 45us (1191us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097902.366728:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1058
00000100:00080000:3.0:1766097903.037770:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766097903.037779:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766097903.037784:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766097903.037794:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2406
00000100:00080000:3.0:1766097903.037798:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766097903.037804:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766097903.037832:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000aadc1282 x1851885432832000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766097903.039105:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000028e7afe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432831936:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097903.039177:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097903.039253:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432831936
00000100:00100000:3.0:1766097903.039319:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1546
00000100:00100000:3.0:1766097903.039333:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d436481 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432831936:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097903.039395:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d436481 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432831936:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (218us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097903.039413:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1546
00000100:00100000:3.0:1766097903.039474:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000028e7afe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432831936:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097907.484804:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417160960
00000100:00100000:3.0:1766097907.484859:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1060
00000100:00100000:3.0:1766097907.484874:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417160960:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097907.484972:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417160960:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 103us (378us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097907.484991:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1060
00000100:00100000:3.0:1766097912.606205:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417161216
00000100:00100000:3.0:1766097912.606255:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1550
00000100:00100000:3.0:1766097912.606263:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009eccc0d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417161216:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097912.606301:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009eccc0d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417161216:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 39us (601us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097912.606308:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1550
00000100:00100000:3.0:1766097913.281069:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009012d88d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432832576:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097913.282230:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097913.282350:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432832576
00000100:00100000:3.0:1766097913.282378:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1551
00000100:00100000:3.0:1766097913.282383:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000028e7afe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432832576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097913.282429:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000028e7afe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432832576:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (199us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097913.282435:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1551
00000100:00100000:3.0:1766097913.282473:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009012d88d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432832576:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097913.282520:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432832640
00000100:00100000:3.0:1766097913.282537:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 916
00000100:00100000:3.0:1766097913.282540:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432832640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766097913.282631:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097913.282657:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432832640:12345-0@lo:8:kworker/u8:3.0 Request processed in 117us (744us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097913.282663:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 916
00000100:00100000:3.0:1766097913.282678:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766097913.282685:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6325418 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432832768:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097913.282703:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097913.282718:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432832768
00000100:00100000:3.0:1766097913.282735:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 795
00000100:00100000:3.0:1766097913.282738:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc163055 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432832768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097913.282760:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc163055 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432832768:12345-0@lo:400:kworker/u8:3.0 Request processed in 21us (57us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097913.282765:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 795
00010000:00080000:3.0:1766097913.282783:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b6325418 x1851885432832768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097920 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766097913.282791:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6325418 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432832768:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766097913.282915:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097913.282919:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097913.282922:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005881774e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432832640:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766097917.726955:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417161408
00000100:00100000:3.0:1766097917.727007:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1553
00000100:00100000:3.0:1766097917.727013:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a105ba27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417161408:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097917.728579:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a105ba27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417161408:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1562us (2537us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097917.728608:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1553
00000100:00100000:3.0:1766097918.397739:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766097918.397777:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a67325cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432832832:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097918.397894:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097918.399076:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432833024
00000100:00100000:3.0:1766097918.399153:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1554
00000100:00100000:3.0:1766097918.399164:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000e55ace7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432833024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097918.399264:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000e55ace7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432833024:12345-0@lo:400:kworker/u8:3.0 Request processed in 99us (709us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097918.399276:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1554
00000100:00100000:3.0:1766097922.845548:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417161600
00000100:00100000:3.0:1766097922.845581:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1556
00000100:00100000:3.0:1766097922.845584:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5d5a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417161600:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097922.847328:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5d5a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417161600:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1736us (2728us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097922.847342:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1556
00000100:00100000:3.0:1766097923.517649:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007bb16575 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432833152:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097923.517725:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097923.517760:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766097923.517769:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dcc52395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432833408:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097923.517808:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097923.517850:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432833408
00000100:00100000:3.0:1766097923.517884:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 797
00000100:00100000:3.0:1766097923.517890:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d05a6a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432833408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097923.518421:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d05a6a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432833408:12345-0@lo:400:kworker/u8:3.0 Request processed in 529us (613us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097923.518429:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 797
00010000:00080000:3.0:1766097923.518459:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000dcc52395 x1851885432833408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097930 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766097923.518471:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dcc52395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432833408:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097923.518604:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b669879a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432833216:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766097927.968357:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417161792
00000100:00100000:3.0:1766097927.968429:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1559
00000100:00100000:3.0:1766097927.968436:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d48a9336 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417161792:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097927.968581:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d48a9336 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417161792:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 144us (757us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097927.968589:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1559
00000100:00080000:3.0:1766097928.641375:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766097928.641384:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766097928.641388:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766097928.641399:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2431
00000100:00080000:3.0:1766097928.641403:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766097928.641410:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766097928.641438:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a5097a7f x1851885432833600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766097928.642018:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0e061f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432833728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097928.642056:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097928.643079:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432833728
00000100:00100000:3.0:1766097928.643117:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432833600
00000100:00100000:3.0:1766097928.643144:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 798
00000100:00100000:3.0:1766097928.643149:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003632e86c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432833728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097928.643190:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003632e86c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432833728:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (1135us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097928.643196:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 798
00000100:00100000:3.0:1766097928.643208:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 922
00000100:00100000:3.0:1766097928.643211:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432833600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766097928.643318:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097928.643354:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432833600:12345-0@lo:8:kworker/u8:0.0 Request processed in 141us (348us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097928.643360:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 922
00010000:00080000:3.0:1766097928.643393:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b0e061f1 x1851885432833728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097935 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097928.643406:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0e061f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432833728:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766097928.643432:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097928.643437:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097928.643440:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5097a7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432833600:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097933.086573:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417161920
00000100:00100000:3.0:1766097933.086643:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1070
00000100:00100000:3.0:1766097933.086651:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417161920:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097933.087996:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417161920:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1344us (2729us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097933.088029:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1070
00000100:00100000:3.0:1766097933.759156:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432834048:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097933.759205:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097933.759244:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432833920:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097933.759269:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097933.759304:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432834048
00000100:00100000:3.0:1766097933.759334:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432833920
00000100:00100000:3.0:1766097933.759352:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 799
00000100:00100000:3.0:1766097933.759358:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5097a7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432834048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097933.759402:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5097a7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432834048:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (199us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097933.759409:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 799
00000100:00100000:3.0:1766097933.759429:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924
00000100:00100000:3.0:1766097933.759432:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0e061f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432833920:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766097933.759567:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097933.759601:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0e061f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432833920:12345-0@lo:8:kworker/u8:0.0 Request processed in 167us (332us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097933.759608:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924
00010000:00080000:3.0:1766097933.759655:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000778ea5b6 x1851885432834048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097940 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097933.759669:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432834048:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766097933.759703:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097933.759707:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097933.759711:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432833920:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097938.878525:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba871264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432834112:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097938.878627:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097938.881168:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2f90738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432834240:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097938.881216:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097938.881254:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432834240
00000100:00100000:3.0:1766097938.881281:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 926
00000100:00100000:3.0:1766097938.881285:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df9b1e46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432834240:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766097938.881408:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097938.881448:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df9b1e46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432834240:12345-0@lo:8:kworker/u8:0.0 Request processed in 161us (232us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097938.881455:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 926
00000100:00080000:3.0:1766097938.881644:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097938.881649:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097938.881653:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2f90738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432834240:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097938.881679:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba871264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432834112:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097938.881690:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766097938.881697:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ae0274d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432834368:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097938.881722:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097938.881746:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432834368
00000100:00100000:3.0:1766097938.881771:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 800
00000100:00100000:3.0:1766097938.881775:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005b204928 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432834368:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097938.881810:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005b204928 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432834368:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (88us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097938.881817:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 800
00010000:00080000:3.0:1766097938.881854:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ae0274d7 x1851885432834368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097945 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097938.881866:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ae0274d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432834368:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097943.325645:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417162368
00000100:00100000:3.0:1766097943.325706:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1568
00000100:00100000:3.0:1766097943.325723:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061081f0b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417162368:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097943.325871:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061081f0b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417162368:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 155us (461us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097943.325889:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1568
00000100:00100000:3.0:1766097943.998085:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1d154b8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432834560:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097943.999195:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097943.999253:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432834560
00010000:00080000:3.0:1766097943.999369:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000077095875 x1851885432834688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097950 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097943.999383:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077095875 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432834688:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097943.999587:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f60d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432834432:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097948.451333:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417162496
00000100:00100000:3.0:1766097949.118313:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432834880
00000100:00100000:3.0:1766097949.118339:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 930
00000100:00100000:3.0:1766097949.118344:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0ecd7ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432834880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766097949.119829:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097949.119879:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0ecd7ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432834880:12345-0@lo:8:kworker/u8:2.0 Request processed in 1534us (1790us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097949.120013:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 930
00000100:00100000:3.0:1766097953.570968:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417162816
00000100:00100000:3.0:1766097953.571811:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 931
00000100:00100000:3.0:1766097953.571817:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f69fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417162816:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766097953.572615:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097953.572670:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f69fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417162816:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 852us (1998us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097953.572679:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 931
00000100:00100000:3.0:1766097958.687250:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417162944
00000100:00100000:3.0:1766097958.687317:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1577
00000100:00100000:3.0:1766097958.687324:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a38773e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417162944:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097958.687409:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a38773e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417162944:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 83us (1996us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097958.687416:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1577
00000100:00100000:3.0:1766097958.688185:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417162880
00000100:00100000:3.0:1766097958.688228:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1080
00000100:00100000:3.0:1766097958.688234:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417162880:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097958.688293:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417162880:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 59us (311us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097958.688301:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1080
00000100:00100000:3.0:1766097959.358232:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432835520
00000100:00100000:3.0:1766097959.358268:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 934
00000100:00100000:3.0:1766097959.358273:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432835520:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766097959.358371:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097959.358929:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432835520:12345-0@lo:8:kworker/u8:0.0 Request processed in 651us (928us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097959.358954:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 934
00000100:00100000:3.0:1766097959.361165:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432835648
00000100:00100000:3.0:1766097959.361222:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 804
00000100:00100000:3.0:1766097959.361227:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dfc1887 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432835648:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097959.361294:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dfc1887 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432835648:12345-0@lo:400:kworker/u8:0.0 Request processed in 66us (337us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097959.361301:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 804
00000100:00100000:3.0:1766097963.804535:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163200
00000100:00100000:3.0:1766097963.804599:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 935
00000100:00100000:3.0:1766097963.804608:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d98167cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417163200:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766097963.804714:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097963.804794:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d98167cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417163200:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 185us (432us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097963.804802:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 935
00000100:00100000:3.0:1766097963.806537:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1082
00000100:00100000:3.0:1766097963.806547:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417163072:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097963.806619:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417163072:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 72us (2322us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097963.806627:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1082
00000100:00100000:3.0:1766097968.930288:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163264
00000100:00100000:3.0:1766097968.930359:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1084
00000100:00100000:3.0:1766097968.930366:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417163264:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097968.930454:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417163264:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 88us (1425us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097968.930490:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1084
00000100:00100000:3.0:1766097968.931949:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163392
00000100:00080000:3.0:1766097969.598567:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766097969.598578:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766097969.598584:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766097969.598594:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2472
00000100:00080000:3.0:1766097969.598598:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766097969.598604:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766097969.598632:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a5097a7f x1851885432836160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766097969.598699:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5097a7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432836160:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766097969.598744:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097969.598974:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432836160
00000100:00100000:3.0:1766097969.599025:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432836288
00000100:00100000:3.0:1766097969.599187:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 806
00000100:00100000:3.0:1766097969.599196:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f60d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432836288:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766097969.599256:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f60d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432836288:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (357us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097969.599265:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 806
00000100:00080000:3.0:1766097969.599314:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097969.599321:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097969.599329:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5097a7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432836160:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766097974.046652:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163456
00000100:00100000:3.0:1766097974.046737:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1086
00000100:00100000:3.0:1766097974.046741:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417163456:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097974.046894:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417163456:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 151us (836us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097974.046902:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1086
00000100:00100000:3.0:1766097974.047181:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163584
00000100:00100000:3.0:1766097974.047212:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 939
00000100:00100000:3.0:1766097974.047217:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dfe02e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417163584:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766097974.047366:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097974.047412:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dfe02e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417163584:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 194us (417us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097974.047419:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 939
00000100:00100000:3.0:1766097974.719824:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000915f289c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432836480:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097974.719873:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097974.719924:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432836480
00000100:00100000:3.0:1766097974.719959:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 940
00000100:00100000:3.0:1766097974.719967:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5097a7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432836480:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766097974.720081:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097974.720136:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5097a7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432836480:12345-0@lo:8:kworker/u8:0.0 Request processed in 167us (263us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097974.720144:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 940
00000100:00100000:3.0:1766097974.720738:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432836608
00000100:00100000:3.0:1766097974.721217:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 807
00000100:00100000:3.0:1766097974.721221:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000677523bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432836608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097974.721258:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000677523bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432836608:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (1036us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097974.721274:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 807
00000100:00100000:3.0:1766097979.170576:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163776
00000100:00100000:3.0:1766097979.170624:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 941
00000100:00100000:3.0:1766097979.170630:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b16f8a66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417163776:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766097979.170735:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097979.170796:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b16f8a66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417163776:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 164us (2347us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097979.170804:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 941
00000100:00100000:3.0:1766097984.287701:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417163840
00000100:00100000:3.0:1766097984.287746:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1090
00000100:00100000:3.0:1766097984.287753:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417163840:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097984.287793:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417163840:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 38us (485us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097984.287800:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1090
00000100:00100000:3.0:1766097984.962141:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000113a6a27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432837248:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097984.962223:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097984.962271:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766097984.962278:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1d154b8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432836992:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097984.962313:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097984.962366:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432837248
00000100:00100000:3.0:1766097984.962422:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 809
00000100:00100000:3.0:1766097984.962428:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fce62d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432837248:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097984.962526:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fce62d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432837248:12345-0@lo:400:kworker/u8:0.0 Request processed in 96us (303us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097984.962556:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 809
00000100:00100000:3.0:1766097984.962597:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432836992
00000100:00100000:3.0:1766097984.962644:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091
00000100:00100000:3.0:1766097984.962650:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432836992:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097984.962684:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432836992:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (371us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097984.962709:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091
00000100:00100000:3.0:1766097984.962783:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1d154b8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432836992:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766097984.962808:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000113a6a27 x1851885432837248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766097991 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097984.962822:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000113a6a27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432837248:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097984.963948:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1593
00000100:00100000:3.0:1766097984.963963:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6c1f26a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432837056:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097984.964249:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6c1f26a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432837056:12345-0@lo:400:kworker/u8:0.0 Request processed in 289us (2475us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097984.964262:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1593
00000100:00100000:3.0:1766097989.408664:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417164032
00000100:00100000:3.0:1766097989.408764:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1092
00000100:00100000:3.0:1766097989.408775:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417164032:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097989.408891:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417164032:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 115us (361us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097989.408903:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1092
00000100:00100000:3.0:1766097990.079157:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1596
00000100:00100000:3.0:1766097990.079224:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d623708b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432837376:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097990.079629:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d623708b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432837376:12345-0@lo:400:kworker/u8:0.0 Request processed in 405us (798us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097990.079660:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1596
00000100:00100000:3.0:1766097994.532982:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417164224
00000100:00100000:3.0:1766097994.533053:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1094
00000100:00100000:3.0:1766097994.533060:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417164224:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766097994.533159:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417164224:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 97us (307us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097994.533168:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1094
00000100:00100000:3.0:1766097995.198769:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002861407e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432837888:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.198800:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097995.198816:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766097995.198821:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6890e1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432837632:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.198846:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766097995.198877:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432837632
00000100:00100000:3.0:1766097995.198901:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095
00000100:00100000:3.0:1766097995.198905:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432837632:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.198930:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432837632:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (85us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097995.198936:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095
00000100:00100000:3.0:1766097995.198973:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6890e1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432837632:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.199015:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432837760
00000100:00100000:3.0:1766097995.199031:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432837888
00000100:00100000:3.0:1766097995.199047:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 811
00000100:00100000:3.0:1766097995.199058:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be9b49bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432837888:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.199086:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be9b49bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432837888:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (287us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097995.199091:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 811
00000100:00100000:3.0:1766097995.199100:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948
00000100:00100000:3.0:1766097995.199103:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432837760:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766097995.199208:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766097995.199238:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432837760:12345-0@lo:8:kworker/u8:0.0 Request processed in 134us (875us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766097995.199244:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948
00010000:00080000:3.0:1766097995.199267:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002861407e x1851885432837888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098002 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766097995.199277:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002861407e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432837888:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.199325:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005492c576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432837824:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766097995.199335:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008faa26e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432837696:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766097995.199376:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766097995.199380:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766097995.199384:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c8141ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432837760:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766097999.645343:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417164416
00000100:00100000:3.0:1766097999.645377:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1096
00000100:00100000:3.0:1766097999.645380:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417164416:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766097999.645920:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417164416:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 537us (755us total) trans 0 rc 0/0
00000100:00100000:3.0:1766097999.645947:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1096
00000100:00100000:3.0:1766098000.319406:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432838016:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098000.321425:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098000.322646:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432838016:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098004.765671:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417164608
00000100:00100000:3.0:1766098004.765714:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1098
00000100:00100000:3.0:1766098004.765720:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417164608:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098004.765875:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417164608:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 153us (855us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098004.765882:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1098
00000100:00100000:3.0:1766098004.768041:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098005.442291:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838272
00000100:00100000:3.0:1766098005.442330:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1099
00000100:00100000:3.0:1766098005.442335:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432838272:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.442378:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432838272:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (226us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098005.442385:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1099
00000100:00100000:3.0:1766098005.442827:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0ecd7ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432838336:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.442856:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098005.442875:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098005.442880:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a61ce132 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432838464:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.442899:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098005.442926:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838336
00000100:00100000:3.0:1766098005.442963:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838464
00000100:00100000:3.0:1766098005.442979:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1605
00000100:00100000:3.0:1766098005.442982:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ee2d640 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432838336:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.443016:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ee2d640 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432838336:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (160us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098005.443022:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1605
00000100:00100000:3.0:1766098005.443032:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1606
00000100:00100000:3.0:1766098005.443035:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b282a1e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432838464:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.443056:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b282a1e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432838464:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (156us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098005.443060:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1606
00000100:00100000:3.0:1766098005.443102:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a61ce132 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432838464:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.443111:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0ecd7ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432838336:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.443147:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f69fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432838400:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098005.443171:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098005.443202:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838528
00000100:00100000:3.0:1766098005.443222:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838400
00000100:00100000:3.0:1766098005.443235:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 813
00000100:00100000:3.0:1766098005.443239:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dfe02e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432838528:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098005.443268:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dfe02e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432838528:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (1185us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098005.443273:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 813
00000100:00100000:3.0:1766098005.443283:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 952
00000100:00100000:3.0:1766098005.443286:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ac22b4d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432838400:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098005.443372:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098005.443410:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ac22b4d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432838400:12345-0@lo:8:kworker/u8:2.0 Request processed in 123us (239us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098005.443417:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 952
00000100:00080000:3.0:1766098005.443445:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098005.443448:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098005.443451:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f69fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432838400:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098009.888168:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417164800
00000100:00100000:3.0:1766098009.888295:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1100
00000100:00100000:3.0:1766098009.888301:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417164800:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098009.888327:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417164800:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 26us (587us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098009.888333:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1100
00000100:00100000:3.0:1766098010.558634:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432838848:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098010.558699:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098010.558735:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098010.558741:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069dabd45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432838592:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098010.558767:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098010.558791:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026329de0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432838656:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098010.558816:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098010.558831:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098010.558835:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432838784:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098010.558857:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098010.558881:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e100b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432838720:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098010.558903:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098010.558951:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838592
00000100:00100000:3.0:1766098010.558986:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101
00000100:00100000:3.0:1766098010.558991:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432838592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098010.559033:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432838592:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (265us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098010.559040:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101
00000100:00100000:3.0:1766098010.559079:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069dabd45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432838592:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098015.009128:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417165120
00000100:00100000:3.0:1766098015.682057:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432838912
00000100:00100000:3.0:1766098015.682103:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103
00000100:00100000:3.0:1766098015.682108:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432838912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098015.682154:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432838912:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (426us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098015.682161:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103
00000100:00100000:3.0:1766098015.683888:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098015.684001:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dfc1887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432838912:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098015.684377:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098015.684382:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098015.684387:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b16f8a66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432839040:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098020.127576:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417165312
00000100:00100000:3.0:1766098020.127608:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 957
00000100:00100000:3.0:1766098020.127614:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ef7b141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417165312:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098020.127717:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098020.129007:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ef7b141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417165312:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1384us (2237us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098020.129041:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 957
00000100:00100000:3.0:1766098020.799836:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0ecd7ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432839232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098020.799939:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098020.802105:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0ecd7ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432839232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098020.802792:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dcc52395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432839424:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098020.804138:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098020.806152:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dcc52395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432839424:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098025.244583:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098025.244626:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098025.244723:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098025.918888:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432839808
00000100:00100000:3.0:1766098025.918921:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432839680
00000100:00100000:3.0:1766098025.918937:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 817
00000100:00100000:3.0:1766098025.918941:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab4d3808 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432839808:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098025.918975:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab4d3808 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432839808:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (945us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098025.918980:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 817
00000100:00100000:3.0:1766098025.918990:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960
00000100:00100000:3.0:1766098025.918998:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432839680:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098025.919099:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098025.919131:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432839680:12345-0@lo:8:kworker/u8:0.0 Request processed in 132us (1021us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098025.919136:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960
00000100:00080000:3.0:1766098025.919188:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098025.919192:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098025.919195:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046201038 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432839680:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098025.919225:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a61ce132 x1851885432839808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098032 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098025.919234:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a61ce132 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432839808:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098030.367439:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417165632
00000100:00100000:3.0:1766098030.368728:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1619
00000100:00100000:3.0:1766098030.368737:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007128d36d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417165632:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098030.368812:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007128d36d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417165632:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (2260us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098030.368820:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1619
00000100:00100000:3.0:1766098031.038330:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008038ff3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432839872:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098031.039175:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098031.039234:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432839872
00000100:00100000:3.0:1766098031.039276:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109
00000100:00100000:3.0:1766098031.039283:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432839872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098031.039331:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432839872:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (156us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098031.039339:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109
00000100:00100000:3.0:1766098031.039396:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008038ff3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432839872:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098031.041338:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098031.041343:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098031.041347:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b8df53e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432840000:0@lo:8:kworker/u8:2.0
00010000:00010000:3.0:1766098033.533734:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098035.486727:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417165760
00000100:00100000:3.0:1766098035.486767:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1110
00000100:00100000:3.0:1766098035.486774:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417165760:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098035.486849:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417165760:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 74us (450us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098035.486857:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1110
00000100:00100000:3.0:1766098035.488044:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098036.158703:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432840320:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098036.159750:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:3.0:1766098036.160130:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098036.160134:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098036.160138:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432840320:0@lo:8:kworker/u8:2.0
00010000:00010000:3.0:1766098036.605625:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098039.677633:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098040.604747:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098040.604799:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098040.604846:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098040.605084:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1112
00000100:00100000:3.0:1766098040.605093:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417165952:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098040.605162:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417165952:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 69us (363us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098040.605176:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1112
00000100:00100000:3.0:1766098041.277652:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f3dc9f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432840512:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098041.277737:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098041.277776:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098041.277784:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8526e9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432840768:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098041.277825:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000536e4c9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432840704:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098041.277860:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098041.277906:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432840512
00000100:00100000:3.0:1766098041.277940:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113
00000100:00100000:3.0:1766098041.277947:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432840512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098041.277993:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432840512:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (259us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098041.277999:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113
00000100:00100000:3.0:1766098041.278040:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432840704
00010000:00010000:3.0:1766098042.749633:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098045.727628:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098045.732170:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098045.732211:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:00010000:3.0:1766098045.821627:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098046.398195:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432840960:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098046.398279:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098046.398332:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432840960
00000100:00100000:3.0:1766098046.398451:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 821
00000100:00100000:3.0:1766098046.398456:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432841088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098046.398622:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432841088:12345-0@lo:400:kworker/u8:0.0 Request processed in 164us (256us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098046.398631:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 821
00010000:00010000:3.0:1766098048.893629:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098051.518202:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432841344
00000100:00100000:3.0:1766098051.518249:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632
00000100:00100000:3.0:1766098051.518255:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a12e35bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432841344:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098051.518305:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a12e35bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432841344:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (249us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098051.518312:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632
00000100:00100000:3.0:1766098051.518347:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432841344:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766098051.520060:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f8aebcd4 x1851885432841408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098058 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098051.520072:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8aebcd4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432841408:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098051.520985:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098051.520991:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098051.520997:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432841280:0@lo:8:kworker/u8:2.0
00010000:00010000:3.0:1766098051.965777:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098055.038521:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098055.965526:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098055.965582:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098055.967303:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098056.638148:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098056.638200:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432841472:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098056.638274:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098056.638309:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432841728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098056.638512:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098056.638553:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432841536
00000100:00100000:3.0:1766098056.638589:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1635
00000100:00100000:3.0:1766098056.638610:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432841536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098056.638656:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432841536:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (336us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098056.638669:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1635
00000100:00100000:3.0:1766098056.638721:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e100b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432841536:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766098056.638852:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000505a1a88 x1851885432841728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098063 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098056.638862:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432841728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098056.638879:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432841472:0@lo:400:kworker/u8:0.0
00010000:00010000:3.0:1766098058.109621:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098061.088733:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417166848
00000100:00100000:3.0:1766098061.088762:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 973
00000100:00100000:3.0:1766098061.088768:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac943495 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417166848:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098061.088862:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098061.088896:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac943495 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417166848:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 127us (253us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098061.088903:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 973
00010000:00010000:3.0:1766098061.181769:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098064.255781:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098066.205784:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417166976
00000100:00100000:3.0:1766098066.205822:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1640
00000100:00100000:3.0:1766098066.205833:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad3cd34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417166976:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098066.206516:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad3cd34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417166976:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 684us (931us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098066.206536:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1640
00010000:00010000:3.0:1766098067.327563:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098069.791939:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:00010000:3.0:1766098070.398555:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098071.327602:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1643
00000100:00100000:3.0:1766098071.327615:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417167232:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098071.327712:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417167232:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 99us (3156us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098071.327723:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1643
00000100:00100000:3.0:1766098071.999684:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432842688
00000100:00100000:3.0:1766098071.999721:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 826
00000100:00100000:3.0:1766098071.999725:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fa0a70e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432842688:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098071.999766:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fa0a70e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432842688:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (150us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098071.999772:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 826
00010000:00010000:3.0:1766098073.469903:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098076.445963:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417167424
00000100:00100000:3.0:1766098076.446046:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1646
00000100:00100000:3.0:1766098076.446053:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1e9dc67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417167424:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098076.446126:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1e9dc67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417167424:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 73us (785us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098076.446135:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1646
00010000:00010000:3.0:1766098076.541774:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098077.118232:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432843008
00000100:00100000:3.0:1766098077.118279:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 827
00000100:00100000:3.0:1766098077.118303:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432843008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098077.118368:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432843008:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (202us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098077.118375:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 827
00000100:00100000:3.0:1766098077.122442:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432842880
00000100:00100000:3.0:1766098077.122506:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980
00000100:00100000:3.0:1766098077.122512:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a9ccd09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432842880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098077.122613:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098077.122719:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a9ccd09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432842880:12345-0@lo:8:kworker/u8:2.0 Request processed in 205us (4335us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098077.122727:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980
00000100:00080000:3.0:1766098077.122788:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098077.122793:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098077.122798:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009557e695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432842880:0@lo:8:kworker/u8:2.0
00010000:00010000:3.0:1766098079.614723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098081.575971:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098081.576033:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098081.576074:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098082.238595:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e23f899b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432843200:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098082.238641:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098082.238697:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432843200
00000100:00100000:3.0:1766098082.239026:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 982
00000100:00100000:3.0:1766098082.239034:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002da19723 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432843200:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098082.239673:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098082.240900:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002da19723 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432843200:12345-0@lo:8:kworker/u8:2.0 Request processed in 1864us (2259us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098082.240923:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 982
00010000:00010000:3.0:1766098082.685690:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098085.757686:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098086.704994:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417167872
00000100:00100000:3.0:1766098086.705055:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 983
00000100:00100000:3.0:1766098086.705063:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e23f899b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417167872:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098086.705178:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098086.705275:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e23f899b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417167872:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 211us (508us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098086.705284:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 983
00000100:00080000:3.0:1766098087.358911:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098087.358921:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098087.358925:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098087.358935:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2590
00000100:00080000:3.0:1766098087.358940:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098087.358956:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098087.358984:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003632e86c x1851885432843520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098087.359035:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003632e86c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432843520:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098087.359081:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098087.362938:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432843520
00000100:00100000:3.0:1766098087.363016:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432843648
00000100:00100000:3.0:1766098087.363416:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984
00000100:00100000:3.0:1766098087.363429:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000964c0684 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432843520:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098087.363548:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098087.363605:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000964c0684 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432843520:12345-0@lo:8:kworker/u8:0.0 Request processed in 175us (4527us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098087.363613:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984
00000100:00080000:3.0:1766098087.363664:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098087.363668:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098087.363673:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003632e86c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432843520:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098087.363728:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c3a048d2 x1851885432843648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098094 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098087.363741:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3a048d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432843648:0@lo:400:kworker/u8:0.0
00010000:00010000:3.0:1766098088.829589:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098091.805998:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098091.806067:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:00010000:3.0:1766098091.901853:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098092.480394:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4af4447 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432843968:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098092.480442:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098092.480500:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098092.480516:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003825ca08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432843712:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098092.480549:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098092.480577:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043193b76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432843840:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098092.480602:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098092.480637:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432843968
00000100:00100000:3.0:1766098092.480669:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432843840
00000100:00100000:3.0:1766098092.480686:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 830
00000100:00100000:3.0:1766098092.480691:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003632e86c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432843968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098092.480732:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003632e86c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432843968:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (290us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098092.480749:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 830
00000100:00100000:3.0:1766098092.480764:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 986
00000100:00100000:3.0:1766098092.480767:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aadc1282 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432843840:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098092.480869:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098092.480912:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aadc1282 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432843840:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (301us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098092.480919:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 986
00000100:00080000:3.0:1766098092.480956:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098092.480960:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098092.480964:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043193b76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432843840:0@lo:8:kworker/u8:2.0
00010000:00080000:3.0:1766098092.480993:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a4af4447 x1851885432843968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098099 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098092.481003:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4af4447 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432843968:0@lo:400:kworker/u8:2.0
00010000:00010000:3.0:1766098094.975632:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098096.925954:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098096.926011:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098096.926059:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098097.598831:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009557e695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432844160:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098097.599258:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098097.601061:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 831
00000100:00100000:3.0:1766098097.601068:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006deaa1cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432844288:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098097.601132:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006deaa1cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432844288:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (2582us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098097.601153:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 831
00000100:00100000:3.0:1766098097.601170:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 988
00000100:00100000:3.0:1766098097.601172:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5097a7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432844160:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098097.601267:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098097.601293:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5097a7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432844160:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (2044us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098097.601298:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 988
00000100:00080000:3.0:1766098097.601326:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098097.601329:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098097.601333:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009557e695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432844160:0@lo:8:kworker/u8:2.0
00010000:00080000:3.0:1766098097.601984:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a4af4447 x1851885432844288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098104 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098097.601997:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4af4447 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432844288:0@lo:400:kworker/u8:2.0
00010000:00010000:3.0:1766098098.045633:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098101.117723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098102.046972:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098102.047024:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098102.050749:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00080000:3.0:1766098102.717762:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098102.717769:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098102.717772:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098102.717779:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2606
00000100:00080000:3.0:1766098102.717782:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098102.717787:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098102.717820:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008faa26e3 x1851885432844480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098102.717890:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005492c576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432844544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.717944:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098102.717974:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098102.717978:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c8141ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432844416:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.717994:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098102.718010:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008faa26e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432844480:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098102.718030:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098102.718059:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432844544
00000100:00100000:3.0:1766098102.718089:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432844416
00000100:00100000:3.0:1766098102.718104:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1662
00000100:00100000:3.0:1766098102.718109:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6890e1a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432844544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.718145:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6890e1a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432844544:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (202us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098102.718152:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1662
00000100:00100000:3.0:1766098102.718172:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1663
00000100:00100000:3.0:1766098102.718175:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000633e633e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432844416:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.718194:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000633e633e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432844416:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (200us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098102.718199:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1663
00000100:00100000:3.0:1766098102.718214:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432844480
00000100:00100000:3.0:1766098102.718227:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 990
00000100:00100000:3.0:1766098102.718230:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432844480:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098102.718335:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098102.718368:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432844480:12345-0@lo:8:kworker/u8:0.0 Request processed in 137us (338us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098102.718374:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 990
00000100:00080000:3.0:1766098102.718404:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098102.718407:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098102.718410:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008faa26e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432844480:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098102.718432:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c8141ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432844416:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.718439:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005492c576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432844544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.720718:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432844352
00000100:00100000:3.0:1766098102.720764:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1137
00000100:00100000:3.0:1766098102.720770:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432844352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098102.720846:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432844352:12345-0@lo:400:kworker/u8:0.0 Request processed in 75us (2219us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098102.720853:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1137
00010000:00010000:3.0:1766098104.189628:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098107.261687:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098107.838112:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b73b081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432844736:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098107.838185:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098107.838224:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098107.838233:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aed9bc0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432844864:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098107.838281:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098107.838340:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f83a089d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432844800:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098107.838376:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098107.838435:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432844736
00000100:00100000:3.0:1766098107.838512:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1665
00000100:00100000:3.0:1766098107.838521:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3ce9c58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432844736:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098107.838578:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3ce9c58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432844736:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (392us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098107.838587:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1665
00010000:00010000:3.0:1766098110.333688:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098112.285613:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098112.285673:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098112.285811:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:00010000:3.0:1766098113.406492:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098116.477631:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098117.410662:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098117.413099:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098117.416409:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00080000:3.0:1766098118.080373:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098118.080382:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098118.080386:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098118.080397:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2621
00000100:00080000:3.0:1766098118.080401:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098118.080408:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098118.080435:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a0dec625 x1851885432845440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098118.081654:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432845312
00000100:00100000:3.0:1766098118.081682:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1143
00000100:00100000:3.0:1766098118.081686:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432845312:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098118.081735:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432845312:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (1092us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098118.081753:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1143
00010000:00010000:3.0:1766098119.550543:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098122.525656:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169088
00000100:00100000:3.0:1766098122.532230:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169216
00000100:00100000:3.0:1766098122.532288:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 997
00000100:00100000:3.0:1766098122.532295:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016cfe613 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417169216:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098122.532987:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098122.533072:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016cfe613 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417169216:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 777us (1082us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098122.533081:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 997
00010000:00010000:3.0:1766098122.621671:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098123.199942:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432845632
00000100:00100000:3.0:1766098123.199982:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1145
00000100:00100000:3.0:1766098123.199987:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432845632:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098123.200038:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432845632:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (434us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098123.200045:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1145
00000100:00100000:3.0:1766098123.200077:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432845888
00000100:00100000:3.0:1766098123.200104:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 836
00000100:00100000:3.0:1766098123.200108:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432845888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098123.200136:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432845888:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (481us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098123.200142:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 836
00000100:00100000:0.0:1766098123.200556:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098123.200724:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432845760
00000100:00100000:3.0:1766098123.200863:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 998
00000100:00100000:3.0:1766098123.200876:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3fa8d43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432845760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098123.200984:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098123.202061:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3fa8d43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432845760:12345-0@lo:8:kworker/u8:2.0 Request processed in 1178us (1499us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098123.202073:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 998
00010000:00010000:3.0:1766098125.693713:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098127.644693:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098127.650098:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098127.650150:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098127.650244:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169344
00000100:00100000:3.0:1766098127.650307:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1676
00000100:00100000:3.0:1766098127.650315:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417169344:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098127.650408:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417169344:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 93us (321us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098127.650417:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1676
00000100:00080000:3.0:1766098128.324051:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098128.324062:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098128.324067:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098128.324077:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2631
00000100:00080000:3.0:1766098128.324081:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098128.324088:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098128.324113:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000cc92587f x1851885432846080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098128.324165:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432846016:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098128.324257:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098128.324299:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098128.324304:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fe70e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432846144:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098128.324325:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098128.324378:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc92587f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432846080:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098128.324411:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098128.324453:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846016
00000100:00100000:3.0:1766098128.324549:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846144
00000100:00100000:3.0:1766098128.324573:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1677
00000100:00100000:3.0:1766098128.324580:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f90ebcbe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432846016:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098128.324635:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f90ebcbe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432846016:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (379us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098128.324656:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1677
00000100:00100000:3.0:1766098128.324674:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1678
00000100:00100000:3.0:1766098128.324679:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2b35baf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432846144:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098128.324704:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2b35baf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432846144:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (378us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098128.324710:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1678
00000100:00100000:3.0:1766098128.324730:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846080
00000100:00100000:3.0:1766098128.324748:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1000
00000100:00100000:3.0:1766098128.324752:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a7d1b9b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432846080:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098128.324850:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098128.324884:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a7d1b9b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432846080:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (473us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098128.324890:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1000
00000100:00080000:3.0:1766098128.325090:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098128.325094:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098128.325098:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc92587f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432846080:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098128.325145:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fe70e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432846144:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098128.325163:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432846016:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098128.327617:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846208
00000100:00100000:0.0:1766098128.327672:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 837
00000100:00100000:0.0:1766098128.327677:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004aed9bc0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432846208:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098128.327723:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004aed9bc0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432846208:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (1064us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098128.327730:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 837
00010000:00010000:3.0:1766098128.765889:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098131.837651:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098132.770164:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169536
00000100:00100000:0.0:1766098132.770225:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1679
00000100:00100000:0.0:1766098132.770232:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000cd1a8d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417169536:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098132.770304:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000cd1a8d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417169536:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 71us (770us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098132.770313:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1679
00000100:00100000:3.0:1766098132.770752:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169472
00000100:00100000:3.0:1766098132.771560:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1148
00000100:00100000:3.0:1766098132.771566:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417169472:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098132.771639:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417169472:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 72us (2193us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098132.771647:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1148
00000100:00080000:0.0:1766098133.438735:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098133.438743:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098133.438746:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098133.438755:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2636
00000100:00080000:0.0:1766098133.438758:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098133.438764:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098133.438786:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008038ff3a x1851885432846400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098133.438838:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b8df53e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432846528:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.438909:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098133.438965:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098133.438971:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000306edddc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432846272:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439002:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098133.439029:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026329de0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432846464:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439056:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098133.439076:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098133.439081:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432846336:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439106:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098133.439143:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846528
00000100:00100000:0.0:1766098133.439188:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 838
00000100:00100000:0.0:1766098133.439195:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432846528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439248:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432846528:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (340us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098133.439262:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 838
00000100:00100000:0.0:1766098133.439291:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846464
00000100:00100000:0.0:1766098133.439318:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846336
00000100:00100000:0.0:1766098133.439340:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680
00000100:00100000:0.0:1766098133.439344:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432846464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439384:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432846464:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (328us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098133.439392:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680
00000100:00100000:0.0:1766098133.439403:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1681
00000100:00100000:0.0:1766098133.439406:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432846336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439448:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432846336:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (341us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098133.439456:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1681
00000100:00100000:0.0:1766098133.439517:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432846336:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098133.439535:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026329de0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432846464:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098133.439786:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008038ff3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432846400:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098133.439820:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766098133.440226:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098133.440231:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098133.440235:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008038ff3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432846400:0@lo:8:kworker/u8:0.0
00010000:00010000:3.0:1766098134.909704:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098137.890164:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169728
00000100:00100000:0.0:1766098137.890350:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098137.890378:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417169792
00000100:00100000:0.0:1766098137.890399:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1003
00000100:00100000:0.0:1766098137.890405:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008038ff3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417169792:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098137.890545:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098137.890615:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008038ff3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417169792:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 210us (267us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098137.890624:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1003
00000100:00100000:3.0:1766098137.891039:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1682
00000100:00100000:3.0:1766098137.891107:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5449de7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417169728:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098137.891352:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5449de7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417169728:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 243us (1426us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098137.891359:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1682
00010000:00010000:3.0:1766098137.981669:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098138.558772:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005d561c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432846848:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.558834:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098138.558873:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098138.558879:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001cfc5d76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432846592:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.558904:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098138.558927:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021f758f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432846784:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.558949:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098138.558963:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098138.558967:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3221592 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432846656:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.558982:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098138.559007:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000076157b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432846720:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559025:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098138.559052:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846592
00000100:00100000:0.0:1766098138.559085:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1151
00000100:00100000:0.0:1766098138.559089:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432846592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559124:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432846592:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (220us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098138.559130:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1151
00000100:00100000:0.0:1766098138.559157:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846784
00000100:00100000:0.0:1766098138.559174:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846656
00000100:00100000:0.0:1766098138.559187:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1683
00000100:00100000:0.0:1766098138.559190:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092fe0f9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432846784:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559214:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092fe0f9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432846784:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (266us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098138.559219:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1683
00000100:00100000:0.0:1766098138.559227:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1684
00000100:00100000:0.0:1766098138.559230:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432846656:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559245:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432846656:12345-0@lo:400:kworker/u8:2.0 Request processed in 15us (262us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098138.559249:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1684
00000100:00100000:0.0:1766098138.559264:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846848
00000100:00100000:0.0:1766098138.559279:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846720
00000100:00100000:0.0:1766098138.559289:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 839
00000100:00100000:0.0:1766098138.559292:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432846848:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559312:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432846848:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (480us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098138.559324:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 839
00000100:00100000:0.0:1766098138.559334:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004
00000100:00100000:0.0:1766098138.559336:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432846720:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098138.559424:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098138.559453:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432846720:12345-0@lo:8:kworker/u8:2.0 Request processed in 115us (427us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098138.559459:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004
00000100:00080000:0.0:1766098138.559537:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098138.559540:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098138.559544:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000076157b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432846720:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559583:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3221592 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432846656:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559591:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021f758f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432846784:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098138.559611:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001cfc5d76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432846592:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766098138.559619:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005d561c86 x1851885432846848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098145 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098138.559627:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005d561c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432846848:0@lo:400:kworker/u8:2.0
00010000:00010000:3.0:1766098141.054948:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098143.007521:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098143.009892:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098143.009942:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098143.678020:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a12e35bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432846912:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098143.678191:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098143.678220:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098143.678226:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ab4d3808 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432847168:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098143.678276:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098143.678434:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846976
00000100:00100000:3.0:1766098143.678612:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1686
00000100:00100000:3.0:1766098143.678618:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d260f26c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432846976:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098143.678659:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d260f26c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432846976:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (607us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098143.678670:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1686
00000100:00100000:0.0:1766098143.678830:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000110879f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432847104:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098143.678865:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098143.678911:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432847040:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098143.678946:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098143.679015:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432846912
00000100:00100000:0.0:1766098143.679041:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1153
00000100:00100000:0.0:1766098143.679045:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432846912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098143.679081:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432846912:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (891us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098143.679088:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1153
00000100:00100000:3.0:1766098143.679536:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432847104
00000100:00100000:3.0:1766098143.679560:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1687
00000100:00100000:3.0:1766098143.679563:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b8df53e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432847104:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098143.679595:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b8df53e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432847104:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (730us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098143.679600:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1687
00000100:00080000:3.0:1766098143.679642:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098143.679646:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098143.679649:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432847040:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098143.679668:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a12e35bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432846912:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098143.679956:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000110879f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432847104:0@lo:400:kworker/u8:0.0
00010000:00010000:3.0:1766098144.126246:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098147.197599:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098148.124745:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098148.124806:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098148.125169:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417170048
00000100:00100000:0.0:1766098148.125248:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1154
00000100:00100000:0.0:1766098148.125255:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417170048:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098148.126161:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417170048:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 904us (1425us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098148.126170:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1154
00000100:00100000:3.0:1766098148.126437:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098148.797960:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432847296
00000100:00100000:0.0:1766098148.798055:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1689
00000100:00100000:0.0:1766098148.798072:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3b0c220 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432847296:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098148.799169:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e25aa539 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432847360:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098148.799251:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098148.800895:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3b0c220 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432847296:12345-0@lo:400:kworker/u8:0.0 Request processed in 2829us (3088us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098148.800903:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1689
00000100:00080000:3.0:1766098148.803517:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098148.803536:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098148.803546:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e25aa539 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432847360:0@lo:8:kworker/u8:0.0
00010000:00010000:3.0:1766098150.269992:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098153.248976:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098153.250661:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098153.250718:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:00010000:3.0:1766098153.341771:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098153.918127:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098153.918134:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098153.918137:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098153.918144:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2657
00000100:00080000:0.0:1766098153.918146:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098153.918151:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098153.918181:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004e100b7b x1851885432847680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098153.918809:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432847808
00000100:00100000:0.0:1766098153.918838:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 842
00000100:00100000:0.0:1766098153.918841:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2941096 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432847808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098153.918873:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2941096 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432847808:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (539us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098153.918878:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 842
00000100:00100000:0.0:1766098153.918894:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432847552
00000100:00100000:0.0:1766098153.918909:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1157
00000100:00100000:0.0:1766098153.918912:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432847552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098153.918931:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432847552:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (530us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098153.918936:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1157
00000100:00100000:0.0:1766098153.918959:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097409eff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432847552:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766098153.918967:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008cce3c11 x1851885432847808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098160 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098153.918974:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008cce3c11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432847808:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098153.919205:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e100b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432847680:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098153.919238:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00010000:3.0:1766098156.413713:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098158.366891:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098158.368971:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098158.369015:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098158.369097:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417170496
00000100:00100000:0.0:1766098158.369157:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1694
00000100:00100000:0.0:1766098158.369162:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c89a32c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417170496:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098158.369228:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c89a32c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417170496:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 65us (258us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098158.369235:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1694
00010000:00010000:3.0:1766098159.485828:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098162.557570:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098163.484273:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098163.488633:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098163.489579:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00080000:0.0:1766098164.157793:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098164.157800:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098164.157802:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098164.157810:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2667
00000100:00080000:0.0:1766098164.157813:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098164.157819:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098164.157821:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432848256:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098164.157837:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d3fa8d43 x1851885432848320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098164.157878:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3fa8d43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432848320:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098164.157885:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098164.158391:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098164.158442:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432848256
00000100:00100000:0.0:1766098164.158517:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1698
00000100:00100000:0.0:1766098164.158525:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002861407e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432848256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098164.159060:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432848256:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098164.159075:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098164.159082:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000953f8749 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432848384:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098164.159111:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098164.159143:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432848384
00000100:00100000:3.0:1766098164.159178:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1699
00000100:00100000:3.0:1766098164.159181:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005492c576 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432848384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098164.159215:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005492c576 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432848384:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (106us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098164.159221:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1699
00000100:00100000:3.0:1766098164.159249:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000953f8749 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432848384:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098164.159275:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432848448
00000100:00100000:3.0:1766098164.159308:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 844
00000100:00100000:3.0:1766098164.159314:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3dc9f44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432848448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098164.159384:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3dc9f44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432848448:12345-0@lo:400:kworker/u8:2.0 Request processed in 70us (135us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098164.159391:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 844
00000100:00100000:0.0:1766098164.159978:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002861407e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432848256:12345-0@lo:400:kworker/u8:2.0 Request processed in 1451us (2091us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098164.159987:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1698
00010000:00010000:3.0:1766098165.629632:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098168.606668:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417170944
00000100:00100000:3.0:1766098168.606721:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1015
00000100:00100000:3.0:1766098168.606726:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000edc6da88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417170944:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098168.606820:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098168.606894:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000edc6da88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417170944:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 167us (2258us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098168.606901:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1015
00000100:00100000:3.0:1766098168.608048:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098168.608140:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098168.608170:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417170816
00000100:00100000:3.0:1766098168.608201:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1162
00000100:00100000:3.0:1766098168.608206:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417170816:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098168.608264:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417170816:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 57us (124us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098168.608271:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1162
00010000:00010000:3.0:1766098168.701762:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098169.278539:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098169.278549:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098169.278554:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098169.278565:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2672
00000100:00080000:3.0:1766098169.278569:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098169.278576:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098169.278604:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c07d779d x1851885432848640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098169.278668:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432848640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098169.278732:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000748960b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432848576:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098169.278758:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098169.279301:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098169.279353:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098169.279365:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000053a6a93f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432848704:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098169.279413:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098169.279469:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432848576
00000100:00100000:3.0:1766098169.279967:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432848512
00000100:00100000:3.0:1766098169.279993:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1163
00000100:00100000:3.0:1766098169.279997:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432848512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098169.280033:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432848512:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (652us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098169.280043:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1163
00000100:00100000:3.0:1766098169.280106:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008faa26e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432848512:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098169.280172:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098169.280178:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098169.280184:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432848640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098169.281849:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1701
00000100:00100000:0.0:1766098169.281859:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000677523bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432848576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098169.281961:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000677523bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432848576:12345-0@lo:400:kworker/u8:2.0 Request processed in 103us (2670us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098169.283867:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000053a6a93f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432848704:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098169.283885:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000748960b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432848576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098169.284238:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1701
00010000:00010000:3.0:1766098171.775568:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098173.725970:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098173.728141:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098173.728175:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098174.399972:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432848832:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098174.400004:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098174.400036:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098174.400041:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007df0c819 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432849088:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098174.400066:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098174.400098:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849088
00000100:00100000:0.0:1766098174.400120:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 846
00000100:00100000:0.0:1766098174.400124:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432849088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098174.400155:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432849088:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (89us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098174.400160:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 846
00010000:00080000:0.0:1766098174.400185:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007df0c819 x1851885432849088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098181 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098174.400194:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007df0c819 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432849088:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098174.400547:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432848832:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098174.400689:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8aebcd4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432848960:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098174.400718:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098174.400795:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432848960
00000100:00100000:0.0:1766098174.400823:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1018
00000100:00100000:0.0:1766098174.400826:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748960b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432848960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098174.400929:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098174.400971:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748960b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432848960:12345-0@lo:8:kworker/u8:0.0 Request processed in 143us (252us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098174.400987:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1018
00010000:00010000:3.0:1766098174.848757:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098177.917753:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098178.846840:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098178.847034:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098178.847135:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171328
00000100:00100000:3.0:1766098178.847205:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1019
00000100:00100000:3.0:1766098178.847214:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097409eff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417171328:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098178.847344:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098178.847431:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097409eff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417171328:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 217us (603us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098178.847439:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1019
00000100:00100000:0.0:1766098178.847934:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171264
00000100:00100000:0.0:1766098178.847975:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1706
00000100:00100000:0.0:1766098178.847979:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8aebcd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417171264:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098178.848026:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8aebcd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417171264:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (227us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098178.848034:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1706
00000100:00100000:3.0:1766098179.518328:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849152
00000100:00100000:3.0:1766098179.518387:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1167
00000100:00100000:3.0:1766098179.518393:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432849152:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098179.518532:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432849216:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098179.518894:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432849280:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098179.519015:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098179.519071:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432849280:12345-0@lo:8:kworker/u8:0.0 Request processed in 176us (362us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098179.519079:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020
00010000:00080000:2.0:1766098179.519136:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009921d012 x1851885432849408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098186 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098179.519149:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432849408:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098179.519312:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432849152:12345-0@lo:400:kworker/u8:0.0 Request processed in 915us (1457us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098179.519321:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1167
00000100:00100000:2.0:1766098179.519408:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432849152:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098179.519472:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098179.520099:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849216
00000100:00100000:0.0:1766098179.521387:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1708
00000100:00100000:0.0:1766098179.521398:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040cd65be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432849216:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098179.521512:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040cd65be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432849216:12345-0@lo:400:kworker/u8:0.0 Request processed in 115us (2042us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098179.521520:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1708
00000100:00100000:0.0:1766098179.521573:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432849216:0@lo:400:kworker/u8:0.0
00010000:00010000:3.0:1766098180.989647:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098183.971288:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098183.971342:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098183.971372:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098183.971520:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171456
00000100:00100000:0.0:1766098183.971582:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1709
00000100:00100000:0.0:1766098183.971589:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d561c86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417171456:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098183.971662:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d561c86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417171456:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 72us (289us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098183.971670:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1709
00000100:00100000:2.0:1766098183.972565:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171392
00000100:00100000:2.0:1766098183.972601:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1168
00000100:00100000:2.0:1766098183.972606:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417171392:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098183.972634:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417171392:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 28us (1294us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098183.972640:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1168
00010000:00010000:3.0:1766098184.062513:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098184.638377:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000053a6a93f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432849472:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098184.638384:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098184.638400:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098184.638403:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098184.638406:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2687
00000100:00080000:3.0:1766098184.638409:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098184.638413:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098184.638431:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008faa26e3 x1851885432849600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098184.638445:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098184.638472:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098184.638504:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007997fad4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432849728:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098184.638529:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098184.638561:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849472
00000100:00100000:2.0:1766098184.638904:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1169
00000100:00100000:2.0:1766098184.638910:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432849472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098184.638958:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432849472:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (514us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098184.638965:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1169
00000100:00100000:2.0:1766098184.639004:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000053a6a93f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432849472:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098184.639716:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849664
00000100:00100000:0.0:1766098184.639744:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849536
00000100:00100000:0.0:1766098184.639761:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1710
00000100:00100000:0.0:1766098184.639765:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000110879f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432849664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098184.639814:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000110879f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432849664:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (389us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098184.639821:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1710
00000100:00100000:0.0:1766098184.639837:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1711
00000100:00100000:0.0:1766098184.639841:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a12e35bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432849536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098184.639866:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a12e35bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432849536:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (370us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098184.639872:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1711
00000100:00100000:0.0:1766098184.640063:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432849536:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098184.640072:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0dec625 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432849664:0@lo:400:kworker/u8:0.0
00010000:00080000:2.0:1766098184.640395:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007997fad4 x1851885432849728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098191 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098184.640405:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007997fad4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432849728:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098184.640966:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098184.640971:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098184.640975:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008faa26e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432849600:0@lo:8:kworker/u8:0.0
00010000:00010000:3.0:1766098187.133626:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098189.086015:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098189.086134:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171584
00000100:00100000:3.0:1766098189.086787:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098189.086874:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098189.087148:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1170
00000100:00100000:2.0:1766098189.087154:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417171584:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098189.087194:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417171584:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 31us (1177us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098189.087200:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1170
00000100:00100000:2.0:1766098189.757663:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:0.0:1766098189.757664:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098189.757681:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000377acd23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432849792:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098189.757697:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffb555d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432849856:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098189.757768:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098189.757769:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766098189.757872:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098189.757876:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098189.757882:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2693
00000100:00100000:2.0:1766098189.757883:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849792
00000100:00080000:0.0:1766098189.757885:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098189.757892:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098189.757930:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f80fd9d7 x1851885432849920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098189.757932:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1171
00000100:00100000:2.0:1766098189.757942:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432849792:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098189.757996:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432849792:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (231us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098189.758001:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f80fd9d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432849920:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098189.758007:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1171
00000100:00100000:0.0:1766098189.758041:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098189.758058:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432850048:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098189.758076:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098189.758085:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ebf710f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432849984:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098189.758091:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098189.758115:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098189.758125:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000377acd23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432849792:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098189.758150:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849856
00000100:00100000:2.0:1766098189.758163:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849920
00000100:00100000:0.0:1766098189.758179:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432849984
00000100:00100000:2.0:1766098189.758203:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850048
00000100:00100000:0.0:1766098189.758236:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1713
00000100:00100000:0.0:1766098189.758241:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0dec625 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432849856:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098189.758275:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0dec625 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432849856:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (510us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098189.758281:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1713
00000100:00100000:0.0:1766098189.758295:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1714
00000100:00100000:0.0:1766098189.758299:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003736a042 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432849984:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098189.758321:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003736a042 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432849984:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (208us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098189.758328:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1714
00000100:00100000:2.0:1766098189.758336:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 849
00000100:00100000:2.0:1766098189.758343:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1995197 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432850048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098189.758365:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ebf710f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432849984:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098189.758381:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffb555d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432849856:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098189.758389:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1995197 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432850048:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (298us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098189.758397:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 849
00000100:00100000:2.0:1766098189.758413:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1024
00000100:00100000:2.0:1766098189.758418:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d91044b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432849920:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098189.758540:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098189.758581:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d91044b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432849920:12345-0@lo:8:kworker/u8:0.0 Request processed in 162us (541us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098189.758587:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1024
00010000:00080000:2.0:1766098189.758713:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003cd96141 x1851885432850048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098196 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098189.758723:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432850048:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098189.759272:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098189.759278:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098189.759283:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f80fd9d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432849920:0@lo:8:kworker/u8:0.0
00010000:00010000:3.0:1766098190.205633:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:3.0:1766098193.277657:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098194.208084:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098194.210305:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098194.210373:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171840
00000100:00100000:0.0:1766098194.210419:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1715
00000100:00100000:0.0:1766098194.210425:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004aed9bc0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417171840:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098194.210504:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004aed9bc0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417171840:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (201us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098194.210512:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1715
00000100:00100000:3.0:1766098194.213607:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098194.213704:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171904
00000100:00100000:2.0:1766098194.213754:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1025
00000100:00100000:2.0:1766098194.213761:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417171904:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098194.213908:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098194.213986:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417171904:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 225us (385us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098194.213994:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1025
00000100:00100000:3.0:1766098194.878053:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006deaa1cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432850112:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098194.878083:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000016cfe613 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432850368:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098194.878297:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098194.878332:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f39038ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432850176:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098194.878353:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098194.878367:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098194.878372:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432850304:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098194.878390:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098194.878414:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432850240:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098194.878424:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098194.878440:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098194.878473:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850112
00000100:00100000:0.0:1766098194.878499:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850368
00000100:00100000:0.0:1766098194.878539:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850240
00000100:00100000:3.0:1766098194.878546:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1173
00000100:00100000:3.0:1766098194.878553:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432850112:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098194.878559:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 850
00000100:00100000:0.0:1766098194.878565:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f80fd9d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432850368:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098194.878605:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432850112:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (183us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098194.878617:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f80fd9d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432850368:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (317us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098194.878617:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1173
00000100:00100000:0.0:1766098194.878627:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 850
00000100:00100000:0.0:1766098194.878645:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1026
00000100:00100000:0.0:1766098194.878648:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000538e714a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432850240:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098194.878660:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006deaa1cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432850112:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098194.878699:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850176
00000100:00100000:3.0:1766098194.878726:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850304
00010000:02020000:0.0:1766098194.878733:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098194.878742:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1716
00000100:00100000:3.0:1766098194.878747:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb555d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432850176:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098194.878781:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000538e714a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432850240:12345-0@lo:8:kworker/u8:2.0 Request processed in 130us (343us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098194.878790:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1026
00000100:00100000:3.0:1766098194.878790:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb555d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432850176:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (434us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098194.878799:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1716
00000100:00100000:3.0:1766098194.878814:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1717
00000100:00100000:3.0:1766098194.878818:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebf710f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432850304:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766098194.878837:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098194.878841:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098194.878844:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432850240:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098194.882065:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebf710f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432850304:12345-0@lo:400:kworker/u8:2.0 Request processed in 3240us (3668us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098194.882080:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1717
00000100:00100000:2.0:1766098194.882226:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f39038ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432850176:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098194.882255:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432850304:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766098194.882553:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000016cfe613 x1851885432850368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098201 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098194.882563:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000016cfe613 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432850368:0@lo:400:kworker/u8:2.0
00010000:00010000:3.0:1766098196.349627:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098199.325662:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098199.325819:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098199.325920:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417171968
00000100:00100000:0.0:1766098199.325985:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1174
00000100:00100000:0.0:1766098199.325994:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417171968:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098199.326086:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417171968:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 92us (435us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098199.326094:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1174
00000100:00100000:3.0:1766098199.331148:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098199.331322:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172096
00000100:00100000:2.0:1766098199.331386:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1027
00000100:00100000:2.0:1766098199.331394:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417172096:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098199.331808:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098199.331902:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417172096:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 506us (766us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098199.331912:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1027
00010000:00010000:3.0:1766098199.421684:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098199.999191:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098199.999198:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098199.999202:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098199.999213:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2703
00000100:00080000:0.0:1766098199.999217:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098199.999223:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098199.999245:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001b13c441 x1851885432850560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098199.999297:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812d02e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432850624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098199.999390:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098199.999423:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098199.999427:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9796e49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432850496:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098199.999446:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098199.999467:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b13c441 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432850560:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098199.999505:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098199.999538:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850624
00000100:00100000:0.0:1766098199.999581:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850496
00000100:00100000:0.0:1766098199.999597:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1719
00000100:00100000:0.0:1766098199.999603:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006141d7f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432850624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098199.999648:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006141d7f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432850624:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (258us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098199.999655:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1719
00000100:00100000:0.0:1766098199.999673:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1720
00000100:00100000:0.0:1766098199.999676:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093fec69d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432850496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098199.999693:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093fec69d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432850496:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (248us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098199.999698:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1720
00000100:00100000:0.0:1766098199.999737:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9796e49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432850496:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098199.999746:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000812d02e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432850624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098200.003008:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850432
00000100:00100000:0.0:1766098200.003086:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1175
00000100:00100000:0.0:1766098200.003095:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432850432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098200.003222:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432850432:12345-0@lo:400:kworker/u8:0.0 Request processed in 127us (2715us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098200.003254:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1175
00000100:00080000:0.0:1766098200.003343:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098200.003349:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098200.003355:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b13c441 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432850560:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098200.003408:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432850432:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098202.508930:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098204.448942:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098204.449581:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098204.450264:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098204.450504:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172160
00000100:00100000:3.0:1766098204.450504:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172288
00000100:00100000:3.0:1766098204.450527:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1029
00000100:00100000:0.0:1766098204.450528:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1176
00000100:00100000:0.0:1766098204.450532:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417172160:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098204.450532:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2941096 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417172288:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:0.0:1766098204.450557:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417172160:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 24us (1622us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098204.450563:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1176
00010000:02020000:3.0:1766098204.450628:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098204.450654:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2941096 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417172288:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 122us (391us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098204.450661:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1029
00000100:00080000:3.0:1766098205.118413:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098205.118420:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098205.118434:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098205.118441:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2708
00000100:00080000:3.0:1766098205.118444:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098205.118449:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098205.118469:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000063dd23a2 x1851885432850880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098205.118556:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fe70e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432850816:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098205.118613:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098205.118641:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098205.118646:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b245c7e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432850944:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098205.118668:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098205.118784:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851008:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098205.118816:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098205.118845:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098205.118850:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432850752:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098205.118875:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098205.118927:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432850752
00000100:00100000:2.0:1766098205.118977:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1177
00000100:00100000:2.0:1766098205.118983:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432850752:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098205.119027:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432850752:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (152us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098205.119043:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1177
00010000:00010000:0.0:1766098205.565681:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098208.637747:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098209.567085:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098209.567124:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098209.567170:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098209.567309:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172352
00000100:00100000:0.0:1766098209.567368:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1178
00000100:00100000:0.0:1766098209.567374:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417172352:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098209.567442:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417172352:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 66us (364us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098209.567451:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1178
00000100:00100000:2.0:1766098209.567724:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172480
00000100:00100000:2.0:1766098209.567757:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1031
00000100:00100000:2.0:1766098209.567761:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748960b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417172480:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098209.567849:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098209.567878:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748960b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417172480:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 116us (708us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098209.567884:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1031
00000100:00080000:3.0:1766098210.239411:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098210.239418:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098210.239422:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098210.239429:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2713
00000100:00080000:3.0:1766098210.239432:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098210.239438:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098210.239459:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c5762093 x1851885432851200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098210.239527:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063dd23a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432851264:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098210.239583:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098210.239623:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098210.239629:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432851136:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098210.239666:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098210.239701:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851264
00000100:00100000:3.0:1766098210.239737:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851136
00000100:00100000:3.0:1766098210.239754:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1725
00000100:00100000:3.0:1766098210.239760:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b245c7e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432851264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098210.239803:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b245c7e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432851264:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (221us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098210.239809:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1725
00000100:00100000:3.0:1766098210.239824:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1726
00000100:00100000:3.0:1766098210.239827:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f180bc23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432851136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098210.239850:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f180bc23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432851136:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (184us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098210.239855:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1726
00000100:00100000:3.0:1766098210.239892:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432851136:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098210.239904:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063dd23a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432851264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098210.240016:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fe70e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851328:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098210.240105:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098210.240148:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098210.240156:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007997fad4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432851072:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098210.240183:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098210.240276:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851328
00000100:00100000:0.0:1766098210.240310:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 853
00000100:00100000:0.0:1766098210.240314:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432851328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098210.240354:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432851328:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (249us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098210.240367:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 853
00010000:00080000:2.0:1766098210.240559:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006fe70e9f x1851885432851328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098217 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098210.240569:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fe70e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851328:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098210.241536:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098210.241542:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098210.241546:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432851200:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098210.241551:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851072
00000100:00100000:3.0:1766098210.241579:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1179
00000100:00100000:3.0:1766098210.241583:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432851072:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098210.242189:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432851072:12345-0@lo:400:kworker/u8:0.0 Request processed in 604us (2005us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098210.242195:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1179
00000100:00100000:2.0:1766098210.242257:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007997fad4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432851072:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098211.709773:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098214.685223:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098214.685279:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098214.685389:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172544
00000100:00100000:0.0:1766098214.685441:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1180
00000100:00100000:0.0:1766098214.685448:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417172544:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098214.685543:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417172544:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 95us (264us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098214.685551:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1180
00000100:00100000:3.0:1766098214.685843:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172608
00000100:00100000:3.0:1766098214.685883:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1727
00000100:00100000:3.0:1766098214.685888:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000edc6da88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417172608:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098214.686026:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000edc6da88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417172608:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 137us (811us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098214.686033:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1727
00000100:00100000:2.0:1766098214.689185:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:00010000:0.0:1766098214.782372:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098215.357657:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098215.357664:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098215.357667:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098215.357694:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2718
00000100:00080000:2.0:1766098215.357697:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098215.357703:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098215.357724:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000021f758f0 x1851885432851520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098215.357780:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000076157b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851648:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.357837:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098215.357860:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098215.357865:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000684dc236 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432851392:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.357888:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098215.357967:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851648
00000100:00100000:0.0:1766098215.358015:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 854
00000100:00100000:0.0:1766098215.358026:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd34a487 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432851648:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.358089:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001cfc5d76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432851456:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.358116:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098215.358133:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098215.358138:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3221592 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432851584:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.358155:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098215.358174:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021f758f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432851520:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098215.358193:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098215.358218:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851456
00000100:00100000:2.0:1766098215.358239:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851584
00000100:00100000:2.0:1766098215.358253:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1728
00000100:00100000:2.0:1766098215.358257:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f83a089d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432851456:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098215.358282:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1729
00000100:00100000:3.0:1766098215.358290:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b73b081 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432851584:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.358293:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f83a089d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432851456:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (174us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098215.358302:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1728
00000100:00100000:2.0:1766098215.358327:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851520
00000100:00100000:3.0:1766098215.358336:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b73b081 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432851584:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (179us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098215.358343:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1729
00000100:00100000:2.0:1766098215.358348:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1034
00000100:00100000:2.0:1766098215.358360:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432851520:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098215.358391:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3221592 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432851584:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098215.358400:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001cfc5d76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432851456:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098215.358437:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851392
00000100:00100000:3.0:1766098215.358458:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1181
00000100:00100000:3.0:1766098215.358461:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432851392:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098215.358657:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd34a487 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432851648:12345-0@lo:400:kworker/u8:0.0 Request processed in 629us (819us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098215.358664:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 854
00000100:00100000:3.0:1766098215.360461:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432851392:12345-0@lo:400:kworker/u8:0.0 Request processed in 1993us (2566us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098215.360486:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1181
00010000:02020000:2.0:1766098215.361257:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098215.361291:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000684dc236 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432851392:0@lo:400:kworker/u8:0.0
00010000:00080000:0.0:1766098215.361306:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000076157b4 x1851885432851648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098222 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098215.361317:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000076157b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851648:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098215.361327:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432851520:12345-0@lo:8:kworker/u8:0.0 Request processed in 2967us (3134us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098215.361335:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1034
00010000:00010000:0.0:1766098217.853871:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098219.806433:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098219.806517:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098219.806551:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098219.806621:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172736
00000100:00100000:0.0:1766098219.806677:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1182
00000100:00100000:0.0:1766098219.806683:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417172736:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098219.806754:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417172736:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 70us (328us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098219.806762:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1182
00000100:00100000:3.0:1766098219.807207:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172800
00000100:00100000:3.0:1766098219.807244:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1730
00000100:00100000:3.0:1766098219.807248:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091386d09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417172800:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098219.807291:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091386d09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417172800:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 41us (774us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098219.807298:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1730
00000100:00100000:3.0:1766098220.480321:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432851712:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098220.480409:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098220.481340:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432851712:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098220.481740:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851904
00000100:00100000:2.0:1766098220.481824:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851968:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098220.481850:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098220.481883:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851840
00000100:00100000:2.0:1766098220.481894:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432851968
00000100:00100000:3.0:1766098220.481903:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1036
00000100:00100000:3.0:1766098220.481907:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432851840:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098220.481914:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 855
00000100:00100000:2.0:1766098220.481917:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021f758f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432851968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098220.481949:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021f758f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432851968:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (99us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098220.481983:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 855
00010000:00080000:2.0:1766098220.482006:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000afdf3ced x1851885432851968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098227 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00010000:02020000:3.0:1766098220.482011:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098220.482015:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432851968:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098220.482044:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432851840:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (344us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098220.482050:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1036
00000100:00100000:0.0:1766098220.482731:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1731
00000100:00100000:0.0:1766098220.482737:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3dc9f44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432851776:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098220.482783:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3dc9f44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432851776:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (1197us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098220.482788:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1731
00000100:00100000:0.0:1766098220.482800:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1732
00000100:00100000:0.0:1766098220.482804:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab4d3808 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432851904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098220.482829:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab4d3808 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432851904:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (1200us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098220.482834:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1732
00000100:00080000:0.0:1766098220.482951:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098220.482968:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098220.482972:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432851840:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098220.483095:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006deaa1cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432851904:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098220.483107:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f39038ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432851776:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098220.925782:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098223.997761:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098224.930598:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098224.930663:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098224.931216:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098224.932379:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417172992
00000100:00100000:3.0:1766098224.932431:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1733
00000100:00100000:3.0:1766098224.932436:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000076157b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417172992:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098224.932504:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000076157b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417172992:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 67us (1920us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098224.932511:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1733
00000100:00100000:0.0:1766098224.933549:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173056
00000100:00100000:0.0:1766098224.933616:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1037
00000100:00100000:0.0:1766098224.933622:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000684dc236 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417173056:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098224.933724:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098224.933792:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000684dc236 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417173056:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 170us (2578us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098224.933801:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1037
00000100:00080000:0.0:1766098225.600072:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098225.600082:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:2.0:1766098225.600085:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b13c441 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432852032:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098225.600086:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098225.600096:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2728
00000100:00080000:0.0:1766098225.600100:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098225.600106:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098225.600133:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c9796e49 x1851885432852160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098225.600182:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098225.600189:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812d02e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432852096:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098225.600232:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098225.600236:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098225.600255:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098225.600261:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000074275f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432852224:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098225.600268:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b413e52a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432852288:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098225.600292:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098225.600312:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098225.600697:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b13c441 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432852032:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098225.606961:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9796e49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432852160:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098225.607032:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098225.607035:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852288
00000100:00100000:2.0:1766098225.607099:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852160
00000100:00100000:2.0:1766098225.607129:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 856
00000100:00100000:2.0:1766098225.607135:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001cfc5d76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432852288:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098225.607162:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1038
00000100:00100000:0.0:1766098225.607168:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc471f6e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432852160:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098225.607192:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001cfc5d76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432852288:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (6881us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098225.607199:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 856
00010000:00080000:2.0:1766098225.607234:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b413e52a x1851885432852288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098232 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098225.607245:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b413e52a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432852288:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766098225.607266:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098225.607325:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc471f6e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432852160:12345-0@lo:8:kworker/u8:0.0 Request processed in 156us (294us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098225.607333:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1038
00000100:00080000:2.0:1766098225.607465:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098225.607471:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098225.607606:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9796e49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432852160:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766098227.069902:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098230.048538:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098230.048593:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098230.048692:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173184
00000100:00100000:2.0:1766098230.048701:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098230.048762:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1736
00000100:00100000:0.0:1766098230.048772:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3221592 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417173184:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098230.049550:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3221592 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417173184:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 775us (953us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098230.049565:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1736
00000100:00100000:3.0:1766098230.049746:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173248
00000100:00100000:3.0:1766098230.049786:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1039
00000100:00100000:3.0:1766098230.049791:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173248:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098230.049914:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098230.049977:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173248:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 185us (1277us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098230.049984:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1039
00010000:00010000:0.0:1766098230.145614:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098230.718034:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432852416:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098230.718118:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098230.718184:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852416
00000100:00100000:0.0:1766098230.718235:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1737
00000100:00100000:0.0:1766098230.718242:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad932d0f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432852416:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098230.718296:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad932d0f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432852416:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (179us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098230.718304:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1737
00000100:00100000:0.0:1766098230.718352:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432852416:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098230.718445:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098230.718456:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003019dda8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432852352:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098230.718529:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098230.718807:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852352
00000100:00100000:0.0:1766098230.718836:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1187
00000100:00100000:0.0:1766098230.718840:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432852352:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098230.718900:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432852352:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (371us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098230.718920:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1187
00000100:00100000:3.0:1766098230.719975:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003019dda8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432852352:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098233.213766:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098235.168120:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098235.168221:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173312
00000100:00100000:0.0:1766098235.168302:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1188
00000100:00100000:0.0:1766098235.168311:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417173312:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098235.168409:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417173312:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 98us (302us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098235.168418:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1188
00000100:00100000:2.0:1766098235.171299:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098235.171339:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098235.171412:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173440
00000100:00100000:0.0:1766098235.171517:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1041
00000100:00100000:0.0:1766098235.171527:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173440:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098235.171644:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098235.171741:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173440:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 213us (401us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098235.171749:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1041
00000100:00080000:0.0:1766098235.837874:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098235.837885:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098235.837891:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098235.837902:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2739
00000100:00080000:0.0:1766098235.837907:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098235.837914:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098235.837942:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000fcd4deb0 x1851885432852800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098235.838004:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9796e49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432852864:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098235.838119:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098235.838215:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852864
00000100:00100000:3.0:1766098235.838243:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b413e52a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432852928:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098235.838285:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098235.838288:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1740
00000100:00100000:2.0:1766098235.838295:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000812d02e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432852864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098235.838311:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098235.838318:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432852672:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098235.838352:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098235.838362:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000812d02e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432852864:12345-0@lo:400:kworker/u8:0.0 Request processed in 67us (245us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098235.838371:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1740
00000100:00100000:3.0:1766098235.838383:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852928
00000100:00100000:0.0:1766098235.838399:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9796e49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432852864:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098235.838410:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 858
00000100:00100000:3.0:1766098235.838415:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432852928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098235.838422:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098235.838429:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ccc08bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432852736:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098235.838460:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098235.838465:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432852928:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (179us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098235.838473:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 858
00010000:00080000:2.0:1766098235.838548:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b413e52a x1851885432852928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098242 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098235.838560:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b413e52a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432852928:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098235.838583:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852736
00000100:00100000:2.0:1766098235.838735:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1741
00000100:00100000:2.0:1766098235.838740:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9796e49 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432852736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098235.838785:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9796e49 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432852736:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (325us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098235.838792:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1741
00000100:00100000:2.0:1766098235.838824:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ccc08bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432852736:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098235.839121:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852800
00000100:00100000:3.0:1766098235.839155:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1042
00000100:00100000:3.0:1766098235.839159:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432852800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098235.839258:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098235.841345:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432852800:12345-0@lo:8:kworker/u8:0.0 Request processed in 2175us (2717us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098235.841360:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1042
00010000:00010000:0.0:1766098236.290113:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098239.358106:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098240.287035:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098240.287095:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098240.287143:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098240.287903:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173504
00000100:00100000:0.0:1766098240.287951:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1190
00000100:00100000:0.0:1766098240.287957:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417173504:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098240.288164:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417173504:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 206us (1069us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098240.288172:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1190
00000100:00100000:3.0:1766098240.288630:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173632
00000100:00100000:3.0:1766098240.288674:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1043
00000100:00100000:3.0:1766098240.288679:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173632:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098240.288782:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098240.288873:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173632:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 193us (1730us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098240.288882:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1043
00000100:00080000:3.0:1766098240.959416:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098240.959424:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098240.959428:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098240.959434:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2744
00000100:00080000:3.0:1766098240.959436:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098240.959443:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098240.959467:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006f2f7702 x1851885432853120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098240.959551:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432853120:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098240.959586:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098240.959657:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853120
00000100:00100000:2.0:1766098240.959689:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432853184:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098240.959701:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853248
00000100:00100000:2.0:1766098240.959729:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098240.959751:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098240.959762:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432853056:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098240.959784:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098240.959824:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044
00000100:00100000:2.0:1766098240.959832:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432853120:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098240.959936:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098240.959993:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 859
00000100:00100000:2.0:1766098240.959997:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432853120:12345-0@lo:8:kworker/u8:0.0 Request processed in 165us (411us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098240.959998:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003019dda8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432853248:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098240.960004:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044
00000100:00100000:2.0:1766098240.960031:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853184
00000100:00100000:2.0:1766098240.960064:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853056
00000100:00100000:2.0:1766098240.960079:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1743
00000100:00100000:2.0:1766098240.960082:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000306edddc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432853184:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098240.960116:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000306edddc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432853184:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (386us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098240.960122:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1743
00000100:00100000:2.0:1766098240.960133:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1744
00000100:00100000:2.0:1766098240.960136:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cce3c11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432853056:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098240.960158:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cce3c11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432853056:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (374us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098240.960164:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1744
00000100:00100000:0.0:1766098240.960181:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432852992
00000100:00080000:2.0:1766098240.960208:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098240.960216:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098240.960221:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432853120:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098240.960253:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432853056:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098240.960268:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432853184:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098240.960460:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003019dda8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432853248:12345-0@lo:400:kworker/u8:0.0 Request processed in 461us (837us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098240.960467:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 859
00000100:00100000:0.0:1766098240.960571:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1191
00000100:00100000:0.0:1766098240.960576:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432852992:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098240.961050:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432852992:12345-0@lo:400:kworker/u8:0.0 Request processed in 473us (1518us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098240.961059:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1191
00010000:00010000:0.0:1766098242.431172:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098245.412300:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173760
00000100:00100000:2.0:1766098245.412374:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1745
00000100:00100000:2.0:1766098245.412382:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f39038ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417173760:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098245.412509:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f39038ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417173760:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 93us (454us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098245.412531:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1745
00000100:00100000:3.0:1766098245.412875:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173696
00000100:00100000:3.0:1766098245.412916:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1192
00000100:00100000:3.0:1766098245.412920:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417173696:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098245.412955:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417173696:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (795us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098245.412971:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1192
00000100:00100000:0.0:1766098245.414678:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173824
00000100:00100000:0.0:1766098245.415330:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1045
00000100:00100000:0.0:1766098245.415339:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173824:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098245.415445:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098245.417167:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417173824:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1820us (5211us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098245.417183:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1045
00010000:00010000:0.0:1766098245.501852:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098246.078207:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098246.078303:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000052d7190 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432853376:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098246.078402:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098246.078587:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000532dc8bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432853312:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098246.078630:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098246.078680:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853312
00000100:00100000:2.0:1766098246.078737:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1193
00000100:00100000:2.0:1766098246.078744:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432853312:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098246.078831:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432853312:12345-0@lo:400:kworker/u8:0.0 Request processed in 87us (200us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098246.078840:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1193
00000100:00100000:2.0:1766098246.078902:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000532dc8bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432853312:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098246.079454:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098246.079463:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098246.079467:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098246.079474:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2749
00000100:00080000:3.0:1766098246.079489:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098246.079494:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098246.079528:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000033ac3a12 x1851885432853440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098246.079590:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088aa54c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432853504:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098246.079632:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098246.079687:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033ac3a12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432853440:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098246.079718:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098246.079754:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853376
00000100:00100000:2.0:1766098246.080101:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853504
00000100:00100000:2.0:1766098246.080124:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1746
00000100:00100000:2.0:1766098246.080129:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074275f18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432853376:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098246.080178:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074275f18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432853376:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (1778us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098246.080185:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1746
00000100:00100000:2.0:1766098246.080197:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1747
00000100:00100000:2.0:1766098246.080200:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b13c441 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432853504:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098246.081040:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098246.081051:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432853568:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098246.081091:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098246.081127:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853440
00000100:00100000:3.0:1766098246.081148:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853568
00000100:00100000:3.0:1766098246.081168:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 860
00000100:00100000:3.0:1766098246.081171:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065002652 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432853568:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098246.081206:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065002652 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432853568:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (115us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098246.081213:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 860
00000100:00100000:3.0:1766098246.081227:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1046
00000100:00100000:3.0:1766098246.081230:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432853440:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098246.081323:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098246.081355:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432853440:12345-0@lo:8:kworker/u8:0.0 Request processed in 124us (1637us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098246.081361:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1046
00000100:00080000:3.0:1766098246.081423:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098246.081427:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098246.081432:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033ac3a12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432853440:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098246.081463:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002c75ca47 x1851885432853568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098253 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098246.081473:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432853568:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098246.081551:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000052d7190 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432853376:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098246.082067:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b13c441 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432853504:12345-0@lo:400:kworker/u8:0.0 Request processed in 1863us (2433us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098246.082087:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1747
00000100:00100000:0.0:1766098246.082333:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088aa54c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432853504:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098246.526014:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098247.549931:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098248.573785:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098249.598343:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098250.526399:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098250.527371:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098250.527413:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098250.527526:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174016
00000100:00100000:1.0:1766098250.527589:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1047
00000100:00100000:1.0:1766098250.527595:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417174016:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098250.527694:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098250.527785:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417174016:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 188us (371us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098250.527793:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1047
00000100:00100000:2.0:1766098250.529550:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173888
00000100:00100000:2.0:1766098250.529627:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1194
00000100:00100000:2.0:1766098250.529634:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417173888:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098250.529724:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417173888:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 90us (2357us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098250.529740:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1194
00000100:00100000:0.0:1766098250.529984:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417173952
00000100:00100000:0.0:1766098250.530027:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1748
00000100:00100000:0.0:1766098250.530031:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000052d7190 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417173952:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098250.530081:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000052d7190 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417173952:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 49us (4005us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098250.530088:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1748
00010000:00010000:1.0:1766098250.621744:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098251.198140:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098251.198150:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098251.198155:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098251.198167:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2754
00000100:00080000:1.0:1766098251.198172:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098251.198180:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098251.198185:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006deaa1cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432853632:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098251.198208:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000afdf3ced x1851885432853760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098251.198270:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008aed407b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432853824:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098251.198280:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098251.198316:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098251.198326:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051a248a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432853888:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098251.198341:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098251.198419:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432853696:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098251.198467:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098251.198565:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853824
00000100:00100000:2.0:1766098251.198565:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432853760:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098251.198623:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1749
00000100:00100000:2.0:1766098251.198623:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098251.198635:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007bb7b14f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432853824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098251.198677:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853632
00000100:00100000:2.0:1766098251.198735:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1195
00000100:00100000:0.0:1766098251.198740:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007bb7b14f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432853824:12345-0@lo:400:kworker/u8:2.0 Request processed in 87us (381us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098251.198748:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432853632:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098251.198765:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1749
00000100:00100000:2.0:1766098251.198815:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432853632:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (541us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098251.198826:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1195
00000100:00100000:0.0:1766098251.198893:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853888
00000100:00100000:0.0:1766098251.198928:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853760
00000100:00100000:0.0:1766098251.198961:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 861
00000100:00100000:0.0:1766098251.198965:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000262a7d82 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432853888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098251.199022:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000262a7d82 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432853888:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (556us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098251.199028:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 861
00000100:00100000:0.0:1766098251.199041:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1048
00000100:00100000:0.0:1766098251.199045:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000953f8749 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432853760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098251.199136:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:2.0:1766098251.199252:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000051a248a0 x1851885432853888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098258 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098251.199267:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051a248a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432853888:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098251.199277:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006deaa1cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432853632:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098251.199385:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000953f8749 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432853760:12345-0@lo:8:kworker/u8:2.0 Request processed in 337us (766us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098251.199393:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1048
00000100:00080000:2.0:1766098251.199721:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098251.199736:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098251.199741:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432853760:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098251.199831:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008aed407b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432853824:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098251.199841:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098251.199989:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853696
00000100:00100000:1.0:1766098251.200634:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1750
00000100:00100000:1.0:1766098251.200641:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ac3a12 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432853696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098251.200723:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ac3a12 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432853696:12345-0@lo:400:kworker/u8:2.0 Request processed in 80us (887us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098251.200755:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1750
00000100:00100000:2.0:1766098251.200884:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432853696:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098251.645769:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098252.669789:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098253.693770:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098254.719061:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098255.645843:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098255.645888:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098255.645933:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098255.645945:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174208
00000100:00100000:0.0:1766098255.646019:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1049
00000100:00100000:0.0:1766098255.646028:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce707d24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417174208:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:2.0:1766098255.646069:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174144
00000100:00100000:2.0:1766098255.646122:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1751
00000100:00100000:2.0:1766098255.646128:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008aed407b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417174144:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00010000:02020000:0.0:1766098255.646138:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098255.646221:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008aed407b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417174144:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 89us (286us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098255.646232:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce707d24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417174208:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 201us (395us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098255.646239:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1751
00000100:00100000:0.0:1766098255.646274:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1049
00000100:00100000:3.0:1766098255.647820:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174080
00000100:00100000:3.0:1766098255.647893:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1196
00000100:00100000:3.0:1766098255.647901:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417174080:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098255.647991:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417174080:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (2103us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098255.647998:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1196
00010000:00010000:1.0:1766098255.742226:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098256.318107:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6eeb4e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432854016:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098256.318204:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098256.318372:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064afdb67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432853952:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098256.318412:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098256.318457:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432853952
00000100:00100000:0.0:1766098256.318524:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1197
00000100:00100000:0.0:1766098256.318532:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432853952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098256.318583:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432853952:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (171us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098256.318591:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1197
00000100:00100000:0.0:1766098256.318640:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064afdb67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432853952:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098256.318887:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098256.318894:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098256.318899:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098256.318907:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2759
00000100:00080000:1.0:1766098256.318910:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098256.318915:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098256.318946:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000576289dc x1851885432854080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098256.319012:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098256.319020:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f9058ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432854144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098256.319054:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098256.319093:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854016
00000100:00100000:2.0:1766098256.319129:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854144
00000100:00100000:2.0:1766098256.319152:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1752
00000100:00100000:2.0:1766098256.319158:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432854016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098256.319209:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432854016:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (1005us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098256.319216:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1752
00000100:00100000:2.0:1766098256.319238:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1753
00000100:00100000:2.0:1766098256.319243:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097409eff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432854144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098256.319275:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097409eff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432854144:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (220us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098256.319282:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1753
00000100:00100000:3.0:1766098256.319300:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000576289dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432854080:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098256.319370:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098256.319444:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f9058ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432854144:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098256.319457:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6eeb4e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432854016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098256.319722:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed4dd325 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432854208:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098256.319766:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098256.319804:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854080
00000100:00100000:1.0:1766098256.319828:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854208
00000100:00100000:1.0:1766098256.319849:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 862
00000100:00100000:1.0:1766098256.319853:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062911c25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432854208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098256.319888:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062911c25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432854208:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (123us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098256.319895:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 862
00000100:00100000:1.0:1766098256.319909:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1050
00000100:00100000:1.0:1766098256.319913:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432854080:12345-0@lo:8:kworker/u8:2.0
00010000:00080000:2.0:1766098256.324406:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ed4dd325 x1851885432854208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098263 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098256.324454:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed4dd325 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432854208:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766098256.340951:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098256.341087:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432854080:12345-0@lo:8:kworker/u8:2.0 Request processed in 21169us (21714us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098256.341105:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1050
00000100:00080000:3.0:1766098256.341313:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098256.341335:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098256.341341:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000576289dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432854080:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766098256.765822:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098257.790772:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098258.814275:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098259.837741:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098260.766979:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098260.767022:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098260.767091:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174336
00000100:00100000:2.0:1766098260.767140:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1754
00000100:00100000:2.0:1766098260.767146:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d5da1ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417174336:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098260.767220:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d5da1ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417174336:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 73us (198us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098260.767226:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1754
00000100:00100000:3.0:1766098260.767863:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174272
00000100:00100000:0.0:1766098260.768728:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1198
00000100:00100000:0.0:1766098260.768735:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417174272:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098260.768794:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417174272:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 60us (1823us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098260.768801:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1198
00000100:00100000:1.0:1766098260.771355:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098260.772809:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174400
00000100:00100000:2.0:1766098260.772856:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1051
00000100:00100000:2.0:1766098260.772862:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac54f788 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174400:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098260.773098:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098260.773424:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac54f788 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174400:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 561us (2075us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098260.773433:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1051
00010000:00010000:0.0:1766098260.861811:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098261.438447:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098261.438454:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098261.438457:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098261.438465:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2764
00000100:00080000:0.0:1766098261.438468:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098261.438472:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098261.438509:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000885bd0b9 x1851885432854400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098261.438602:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432854528:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.438677:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098261.438716:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098261.438722:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f80fd9d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432854272:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.438747:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098261.438787:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc471f6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432854464:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.438812:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098261.438829:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098261.438834:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432854336:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.438852:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098261.438874:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432854400:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098261.438895:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098261.438927:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854464
00000100:00100000:3.0:1766098261.438969:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854336
00000100:00100000:3.0:1766098261.438987:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1755
00000100:00100000:3.0:1766098261.438991:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432854464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439034:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432854464:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (222us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098261.439046:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1755
00000100:00100000:3.0:1766098261.439060:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1756
00000100:00100000:3.0:1766098261.439064:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065002652 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432854336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439087:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065002652 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432854336:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (234us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098261.439101:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1756
00000100:00100000:3.0:1766098261.439128:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854528
00000100:00100000:3.0:1766098261.439145:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854400
00000100:00100000:3.0:1766098261.439159:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 863
00000100:00100000:3.0:1766098261.439163:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432854528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439193:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432854528:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (516us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098261.439199:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 863
00000100:00100000:3.0:1766098261.439208:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052
00000100:00100000:3.0:1766098261.439212:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432854400:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098261.439303:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098261.439333:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432854400:12345-0@lo:8:kworker/u8:0.0 Request processed in 120us (437us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098261.439339:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052
00000100:00080000:3.0:1766098261.439371:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098261.439375:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098261.439379:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432854400:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439401:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854272
00000100:00100000:3.0:1766098261.439419:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1199
00000100:00100000:3.0:1766098261.439422:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432854272:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439444:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432854272:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (698us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098261.439449:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1199
00000100:00100000:3.0:1766098261.439509:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432854336:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439529:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc471f6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432854464:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098261.439550:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f80fd9d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432854272:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766098261.439557:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000262a7d82 x1851885432854528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098268 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098261.439564:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432854528:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098261.886544:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098262.909908:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098263.933758:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098264.957969:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098265.886276:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098265.888527:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174592
00000100:00100000:0.0:1766098265.888609:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1053
00000100:00100000:0.0:1766098265.888624:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174592:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098265.888729:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098265.890284:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174592:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1661us (4009us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098265.890298:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1053
00000100:00100000:2.0:1766098265.890590:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098265.890614:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098265.891752:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174464
00000100:00100000:0.0:1766098265.891804:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1200
00000100:00100000:0.0:1766098265.891810:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417174464:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098265.892100:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417174464:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 289us (1511us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098265.892107:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1200
00000100:00100000:3.0:1766098265.893407:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174528
00000100:00100000:3.0:1766098265.893462:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1757
00000100:00100000:3.0:1766098265.893466:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417174528:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098265.895301:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417174528:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1824us (4677us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098265.895318:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1757
00010000:00010000:1.0:1766098265.983207:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098266.557737:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098266.557745:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098266.557748:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098266.557757:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2769
00000100:00080000:0.0:1766098266.557760:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098266.557772:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098266.557795:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008d91044b x1851885432854720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098266.557846:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432854656:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098266.557908:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098266.557954:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098266.557961:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d91044b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432854720:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098266.557963:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000377acd23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432854784:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098266.558003:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098266.558009:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098266.558039:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854656
00000100:00100000:1.0:1766098266.558069:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854720
00000100:00100000:0.0:1766098266.558075:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854784
00000100:00100000:1.0:1766098266.558100:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1054
00000100:00100000:0.0:1766098266.558103:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1758
00000100:00100000:1.0:1766098266.558109:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000667f5d6e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432854720:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098266.558110:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f80fd9d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432854656:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766098266.558211:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098266.558250:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000667f5d6e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432854720:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (245us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098266.558257:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1054
00000100:00080000:1.0:1766098266.558294:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098266.558305:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098266.558309:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d91044b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432854720:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098266.558359:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1759
00000100:00100000:1.0:1766098266.558364:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc471f6e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432854784:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098266.558394:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003644f3ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432854592:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098266.558408:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc471f6e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432854784:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (404us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098266.558414:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1759
00000100:00100000:2.0:1766098266.558432:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098266.558457:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098266.558463:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432854848:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098266.558473:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000377acd23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432854784:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098266.558505:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098266.558530:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854592
00000100:00100000:2.0:1766098266.558542:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854848
00000100:00100000:1.0:1766098266.558555:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1201
00000100:00100000:1.0:1766098266.558559:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432854592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098266.558581:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 864
00000100:00100000:2.0:1766098266.558586:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432854848:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098266.558596:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432854592:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (163us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098266.558603:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1201
00000100:00100000:2.0:1766098266.558641:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432854848:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (136us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098266.558648:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 864
00010000:00080000:2.0:1766098266.558678:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000262a7d82 x1851885432854848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098273 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098266.558686:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000262a7d82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432854848:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098266.558693:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003644f3ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432854592:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098266.562560:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f80fd9d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432854656:12345-0@lo:400:kworker/u8:2.0 Request processed in 4441us (4644us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098266.562576:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1758
00000100:00100000:1.0:1766098266.562692:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432854656:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098267.011599:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098268.029724:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098269.053662:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098270.078376:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098271.004614:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098271.004665:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098271.004756:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098271.004880:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174784
00000100:00100000:1.0:1766098271.004943:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1055
00000100:00100000:1.0:1766098271.004954:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174784:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098271.005072:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098271.005159:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174784:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 207us (557us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098271.005217:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1055
00000100:00100000:3.0:1766098271.005411:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174656
00000100:00100000:3.0:1766098271.005449:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1202
00000100:00100000:3.0:1766098271.005453:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417174656:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098271.005604:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174720
00000100:00100000:0.0:1766098271.005634:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1760
00000100:00100000:0.0:1766098271.005638:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000953f8749 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417174720:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098271.005976:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417174656:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 520us (1219us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098271.005984:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1202
00000100:00100000:0.0:1766098271.006671:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000953f8749 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417174720:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1029us (2002us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098271.006681:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1760
00010000:00010000:1.0:1766098271.103848:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098271.677756:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432854912:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766098271.677757:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098271.677766:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098271.677769:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098271.677774:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2775
00000100:00080000:0.0:1766098271.677777:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098271.677786:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098271.677811:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000fcd4deb0 x1851885432855040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098271.677865:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098271.677897:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098271.677898:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432855040:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098271.677901:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ccc08bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432855168:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098271.677932:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098271.677971:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854912
00000100:00100000:0.0:1766098271.678028:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1203
00000100:00100000:0.0:1766098271.678036:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432854912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098271.678089:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432854912:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (225us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098271.678096:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1203
00000100:00100000:2.0:1766098271.678190:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432854912:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098271.678225:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855168
00000100:00100000:2.0:1766098271.678253:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 865
00000100:00100000:2.0:1766098271.678256:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432855168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098271.678288:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432855168:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (356us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098271.678293:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 865
00010000:00080000:2.0:1766098271.678327:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003ccc08bd x1851885432855168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098278 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098271.678337:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ccc08bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432855168:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098271.678555:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432854976:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098271.678604:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098271.678623:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098271.678628:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432855104:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098271.678645:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098271.678672:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432854976
00000100:00100000:1.0:1766098271.678697:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855104
00000100:00100000:1.0:1766098271.678712:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1761
00000100:00100000:1.0:1766098271.678715:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003eba2c8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432854976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098271.678743:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003eba2c8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432854976:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (140us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098271.678748:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1761
00000100:00100000:1.0:1766098271.678762:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1762
00000100:00100000:1.0:1766098271.678764:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b222828 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432855104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098271.678780:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b222828 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432855104:12345-0@lo:400:kworker/u8:2.0 Request processed in 15us (136us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098271.678785:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1762
00000100:00100000:1.0:1766098271.678811:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432855104:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098271.679357:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432854976:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098271.679415:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098271.679507:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855040
00000100:00100000:2.0:1766098271.679866:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056
00000100:00100000:2.0:1766098271.679871:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b413e52a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432855040:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766098271.679972:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098271.680860:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b413e52a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432855040:12345-0@lo:8:kworker/u8:2.0 Request processed in 982us (1440us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098271.680872:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056
00000100:00080000:3.0:1766098271.680922:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098271.680929:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098271.680934:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432855040:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766098272.126015:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098273.150552:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098274.173815:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098275.199944:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098276.127012:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098276.128922:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098276.128973:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098276.129078:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174848
00000100:00100000:0.0:1766098276.129166:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1204
00000100:00100000:0.0:1766098276.129176:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417174848:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098276.129269:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417174848:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 93us (2267us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098276.129278:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1204
00000100:00100000:3.0:1766098276.129383:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174912
00000100:00100000:3.0:1766098276.129430:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1763
00000100:00100000:3.0:1766098276.129435:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016cfe613 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417174912:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098276.129513:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016cfe613 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417174912:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 76us (600us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098276.129520:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1763
00000100:00100000:1.0:1766098276.129603:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417174976
00000100:00100000:1.0:1766098276.129641:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1057
00000100:00100000:1.0:1766098276.129646:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a79ed326 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174976:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098276.129749:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098276.129791:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a79ed326 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417174976:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 143us (817us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098276.129798:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1057
00010000:00010000:0.0:1766098276.223057:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098276.797830:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098276.797839:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098276.797843:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098276.797854:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2780
00000100:00080000:1.0:1766098276.797858:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098276.797865:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098276.797893:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000444deb29 x1851885432855360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098276.797953:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4d39d96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432855424:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.798035:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098276.798079:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098276.798084:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025358442 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432855296:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.798104:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098276.798144:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855424
00000100:00100000:1.0:1766098276.798192:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855296
00000100:00100000:1.0:1766098276.798229:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1764
00000100:00100000:1.0:1766098276.798236:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec954843 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432855424:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.798291:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec954843 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432855424:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (257us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098276.798300:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1764
00000100:00100000:1.0:1766098276.798317:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1765
00000100:00100000:1.0:1766098276.798321:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1a3900e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432855296:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.798369:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1a3900e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432855296:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (265us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098276.798376:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1765
00000100:00100000:1.0:1766098276.798425:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000025358442 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432855296:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.798436:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4d39d96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432855424:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098276.798967:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a46965fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432855232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098276.799010:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098276.799030:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098276.799035:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ccdd40b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432855488:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098276.799061:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098276.799987:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855232
00000100:00100000:0.0:1766098276.800037:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1205
00000100:00100000:0.0:1766098276.800042:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432855232:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098276.800104:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432855232:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (1093us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098276.800112:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1205
00000100:00100000:1.0:1766098276.802427:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855488
00000100:00100000:1.0:1766098276.802524:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 866
00000100:00100000:1.0:1766098276.802531:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432855488:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.802637:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432855488:12345-0@lo:400:kworker/u8:0.0 Request processed in 106us (3575us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098276.802661:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 866
00000100:00100000:1.0:1766098276.802728:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432855360:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098276.803445:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006ccdd40b x1851885432855488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098283 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098276.803463:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ccdd40b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432855488:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098276.803491:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a46965fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432855232:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098276.803580:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098276.803677:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855360
00000100:00100000:0.0:1766098276.803711:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1058
00000100:00100000:0.0:1766098276.803716:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f4d39d96 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432855360:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098276.804023:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098276.804393:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f4d39d96 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432855360:12345-0@lo:8:kworker/u8:0.0 Request processed in 675us (813us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098276.804400:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1058
00000100:00080000:1.0:1766098276.804456:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098276.804461:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098276.804465:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432855360:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766098277.245829:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098278.271471:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098279.294400:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098280.318885:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098281.246923:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098281.246968:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098281.247010:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098281.247125:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175104
00000100:00100000:1.0:1766098281.247198:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1766
00000100:00100000:1.0:1766098281.247206:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175104:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098281.247315:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5823b39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175104:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 109us (347us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098281.247343:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1766
00000100:00100000:0.0:1766098281.248020:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175040
00000100:00100000:0.0:1766098281.248059:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1206
00000100:00100000:0.0:1766098281.248063:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175040:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098281.248102:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175040:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 38us (1190us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098281.248107:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1206
00000100:00100000:3.0:1766098281.248470:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175168
00000100:00100000:3.0:1766098281.249154:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1059
00000100:00100000:3.0:1766098281.249161:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009921d012 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417175168:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098281.249264:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098281.249315:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009921d012 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417175168:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 154us (2306us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098281.249323:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1059
00010000:00010000:1.0:1766098281.343047:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098281.917721:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098281.917723:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432855552:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098281.917730:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098281.917734:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098281.917739:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2785
00000100:00080000:1.0:1766098281.917741:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098281.917749:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098281.917772:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000940b152d x1851885432855680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098281.917811:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1fe4f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432855744:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098281.917818:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098281.917849:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098281.917851:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098281.917856:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3b84863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432855808:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098281.917867:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098281.917871:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025358442 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432855616:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098281.917878:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098281.917890:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098281.917909:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000940b152d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432855680:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098281.917916:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855552
00000100:00100000:1.0:1766098281.917932:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098281.917959:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855744
00000100:00100000:2.0:1766098281.917965:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1207
00000100:00100000:2.0:1766098281.917973:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432855552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098281.917985:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855616
00000100:00100000:1.0:1766098281.918000:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1767
00000100:00100000:1.0:1766098281.918004:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009012d88d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432855744:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098281.918022:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432855552:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (205us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098281.918034:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009012d88d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432855744:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (185us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098281.918041:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1767
00000100:00100000:2.0:1766098281.918046:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1207
00000100:00100000:1.0:1766098281.918057:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1768
00000100:00100000:1.0:1766098281.918066:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005881774e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432855616:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098281.918090:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005881774e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432855616:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (200us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098281.918095:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1768
00000100:00100000:2.0:1766098281.918097:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432855552:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098281.918127:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000025358442 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432855616:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098281.918136:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1fe4f18 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432855744:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098281.918231:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855808
00000100:00100000:0.0:1766098281.918273:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855680
00000100:00100000:0.0:1766098281.918299:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 867
00000100:00100000:0.0:1766098281.918303:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecb38511 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432855808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098281.918350:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecb38511 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432855808:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (471us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098281.918355:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 867
00000100:00100000:0.0:1766098281.918369:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1060
00000100:00100000:0.0:1766098281.918372:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a46965fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432855680:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098281.918466:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098281.918546:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a46965fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432855680:12345-0@lo:8:kworker/u8:2.0 Request processed in 173us (614us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098281.918553:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1060
00000100:00080000:0.0:1766098281.918577:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098281.918589:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098281.918592:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000940b152d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432855680:0@lo:8:kworker/u8:2.0
00010000:00080000:2.0:1766098281.918726:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c3b84863 x1851885432855808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098288 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098281.918736:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3b84863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432855808:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098282.365911:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098283.393773:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098284.415206:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098285.437879:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098286.365874:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098286.365930:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098286.366884:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098286.367011:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175296
00000100:00100000:2.0:1766098286.367090:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1769
00000100:00100000:2.0:1766098286.367100:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1fe4f18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417175296:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098286.367203:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1fe4f18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417175296:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 103us (1272us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098286.367213:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1769
00000100:00100000:0.0:1766098286.369269:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175232
00000100:00100000:0.0:1766098286.369356:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1208
00000100:00100000:0.0:1766098286.369364:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175232:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098286.369449:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175232:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 85us (3588us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098286.369456:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1208
00000100:00100000:0.0:1766098286.369655:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175360
00000100:00100000:0.0:1766098286.369679:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1061
00000100:00100000:0.0:1766098286.369684:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025358442 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417175360:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098286.369786:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098286.369819:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025358442 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417175360:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 135us (2940us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098286.369825:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1061
00010000:00010000:1.0:1766098286.462891:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098287.040598:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098287.040607:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098287.040610:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098287.040620:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2790
00000100:00080000:0.0:1766098287.040624:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098287.040631:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098287.040657:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ff866b23 x1851885432856000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098287.040738:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432856128:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.040821:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098287.040860:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098287.040867:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432855872:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.040896:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098287.040922:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432855936:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.040954:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098287.040985:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098287.040992:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ccc08bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432856064:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.041022:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098287.041030:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855936
00000100:00100000:0.0:1766098287.041058:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856000:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098287.041087:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856064
00000100:00100000:0.0:1766098287.041091:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098287.041129:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1770
00000100:00100000:3.0:1766098287.041140:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003644f3ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432855936:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.041149:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432855872
00000100:00100000:0.0:1766098287.041195:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1209
00000100:00100000:0.0:1766098287.041202:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432855872:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098287.041261:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003644f3ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432855936:12345-0@lo:400:kworker/u8:0.0 Request processed in 122us (306us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098287.041271:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1770
00000100:00100000:0.0:1766098287.041277:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432855872:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (378us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098287.041290:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1209
00000100:00100000:3.0:1766098287.041302:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1771
00000100:00100000:3.0:1766098287.041309:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000262a7d82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432856064:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.041334:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856128
00000100:00100000:3.0:1766098287.041361:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000262a7d82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432856064:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (338us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098287.041365:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856000
00000100:00100000:3.0:1766098287.041370:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1771
00000100:00100000:0.0:1766098287.041391:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 868
00000100:00100000:0.0:1766098287.041397:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432856128:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098287.041435:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ccc08bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432856064:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.041443:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432856128:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (622us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098287.041452:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 868
00000100:00100000:3.0:1766098287.041456:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432855936:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098287.041519:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1062
00000100:00100000:0.0:1766098287.041523:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000377acd23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432856000:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098287.041614:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098287.041632:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432855872:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766098287.041662:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000538e714a x1851885432856128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098294 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098287.041663:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000377acd23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432856000:12345-0@lo:8:kworker/u8:0.0 Request processed in 134us (570us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098287.041685:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1062
00000100:00100000:3.0:1766098287.041696:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432856128:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098287.041735:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098287.041740:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098287.041746:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856000:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766098287.487223:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098288.510062:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098289.534159:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098290.560638:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098291.485373:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098291.485463:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098291.485534:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175424
00000100:00100000:2.0:1766098291.485541:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098291.485601:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1210
00000100:00100000:3.0:1766098291.485609:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175424:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098291.485699:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175488
00000100:00100000:1.0:1766098291.485763:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1772
00000100:00100000:1.0:1766098291.485768:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003019dda8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175488:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098291.487455:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175552
00000100:00100000:3.0:1766098291.489149:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175424:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 3528us (3675us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098291.489169:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1210
00000100:00100000:1.0:1766098291.489309:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003019dda8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175488:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 3537us (3765us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098291.489320:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1772
00000100:00100000:2.0:1766098291.490074:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1063
00000100:00100000:2.0:1766098291.490083:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417175552:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098291.490201:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098291.490270:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417175552:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 186us (4909us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098291.490278:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1063
00010000:00010000:0.0:1766098291.582405:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098292.158045:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098292.158055:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098292.158059:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098292.158070:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2795
00000100:00080000:0.0:1766098292.158074:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098292.158082:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098292.158110:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000538e714a x1851885432856320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098292.158219:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d91044b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432856256:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098292.158303:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098292.158337:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098292.158343:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432856384:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098292.158367:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098292.158414:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856320:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098292.158499:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098292.158583:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432856192:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098292.158629:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098292.158633:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856256
00000100:00100000:2.0:1766098292.158665:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098292.158672:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432856448:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098292.158680:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856384
00000100:00100000:0.0:1766098292.158702:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1773
00000100:00100000:2.0:1766098292.158707:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098292.158713:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ccc08bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432856256:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098292.158742:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856192
00000100:00100000:0.0:1766098292.158780:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ccc08bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432856256:12345-0@lo:400:kworker/u8:0.0 Request processed in 68us (478us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098292.158784:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1211
00000100:00100000:2.0:1766098292.158795:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432856192:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098292.158796:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1773
00000100:00100000:0.0:1766098292.158820:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1774
00000100:00100000:0.0:1766098292.158825:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2941096 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432856384:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098292.158842:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432856192:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (212us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098292.158851:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1211
00000100:00100000:0.0:1766098292.158857:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2941096 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432856384:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (488us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098292.158864:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1774
00000100:00100000:2.0:1766098292.158891:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856320
00000100:00100000:2.0:1766098292.158918:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856448
00000100:00100000:2.0:1766098292.158946:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 869
00000100:00100000:2.0:1766098292.158949:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432856448:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098292.158985:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432856448:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (281us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098292.158992:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 869
00000100:00100000:2.0:1766098292.159007:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064
00000100:00100000:2.0:1766098292.159011:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011bca332 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432856320:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098292.159046:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432856384:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098292.159054:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d91044b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432856256:0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766098292.159117:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098292.159157:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011bca332 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432856320:12345-0@lo:8:kworker/u8:0.0 Request processed in 144us (680us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098292.159164:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064
00000100:00080000:2.0:1766098292.159205:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098292.159209:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098292.159213:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000538e714a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856320:0@lo:8:kworker/u8:0.0
00010000:00080000:2.0:1766098292.159241:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000885bd0b9 x1851885432856448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098299 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098292.159257:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000885bd0b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432856448:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098292.159267:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff866b23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432856192:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098292.605763:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098293.631209:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098294.654339:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098295.677855:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098296.606816:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098296.606868:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098296.606951:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098296.607001:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175616
00000100:00100000:3.0:1766098296.607043:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175680
00000100:00100000:1.0:1766098296.607091:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1212
00000100:00100000:3.0:1766098296.607113:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1775
00000100:00100000:1.0:1766098296.607121:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175616:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098296.607134:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000032a0d422 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175680:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098296.607251:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175616:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 137us (382us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098296.607266:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1212
00000100:00100000:0.0:1766098296.609733:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175744
00000100:00100000:0.0:1766098296.609803:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1065
00000100:00100000:0.0:1766098296.609810:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000532dc8bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417175744:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098296.609912:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098296.610030:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000032a0d422 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175680:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 2901us (3078us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098296.610041:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1775
00000100:00100000:0.0:1766098296.610556:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000532dc8bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417175744:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 744us (3748us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098296.610565:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1065
00010000:00010000:1.0:1766098296.704183:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098297.282648:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098297.282655:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098297.282658:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098297.282666:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2800
00000100:00080000:3.0:1766098297.282669:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098297.282675:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098297.282697:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b4d81768 x1851885432856640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098297.282790:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098297.282808:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432856576:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098297.282881:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098297.282932:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098297.282939:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432856512:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098297.282968:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098297.283001:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856512
00000100:00100000:1.0:1766098297.283051:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1213
00000100:00100000:2.0:1766098297.283051:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856576
00000100:00100000:1.0:1766098297.283061:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432856512:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098297.283123:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432856512:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (150us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098297.283139:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1776
00000100:00100000:1.0:1766098297.283141:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1213
00000100:00100000:2.0:1766098297.283152:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ccdd40b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432856576:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098297.283202:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432856512:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098297.283255:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ccdd40b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432856576:12345-0@lo:400:kworker/u8:0.0 Request processed in 104us (372us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098297.283266:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1776
00000100:00100000:1.0:1766098297.283772:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432856576:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098297.283910:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432856768:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098297.283946:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098297.283979:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856640:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098297.284002:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098297.284029:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856768
00000100:00100000:3.0:1766098297.284056:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856640
00000100:00100000:3.0:1766098297.284070:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 870
00000100:00100000:3.0:1766098297.284074:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c57c659 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432856768:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098297.284112:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c57c659 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432856768:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (166us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098297.284118:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 870
00000100:00100000:3.0:1766098297.284146:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1066
00000100:00100000:3.0:1766098297.284149:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432856640:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098297.284239:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098297.284272:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432856640:12345-0@lo:8:kworker/u8:0.0 Request processed in 121us (270us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098297.284278:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1066
00000100:00080000:3.0:1766098297.284303:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098297.284307:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098297.284310:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856640:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098297.284331:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000006cd3ba3 x1851885432856768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098304 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098297.284340:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432856768:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098297.284413:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432856704:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098297.284441:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098297.286598:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856704
00000100:00100000:3.0:1766098297.287133:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1777
00000100:00100000:3.0:1766098297.287141:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000576289dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432856704:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098297.287873:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000576289dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432856704:12345-0@lo:400:kworker/u8:0.0 Request processed in 730us (3429us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098297.287882:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1777
00000100:00100000:1.0:1766098297.287976:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432856704:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098297.725675:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098298.749722:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098299.773768:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098300.797845:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098301.725563:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098301.725627:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098301.725646:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175936
00000100:00100000:2.0:1766098301.725710:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1067
00000100:00100000:2.0:1766098301.725718:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417175936:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:3.0:1766098301.725732:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098301.725791:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175872
00010000:02020000:2.0:1766098301.725832:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098301.725847:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1778
00000100:00100000:3.0:1766098301.725854:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748960b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175872:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098301.725901:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417175936:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 183us (350us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098301.725911:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1067
00000100:00100000:3.0:1766098301.725960:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748960b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417175872:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 101us (236us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098301.725985:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1778
00000100:00100000:1.0:1766098301.726368:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417175808
00000100:00100000:2.0:1766098301.727104:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1214
00000100:00100000:2.0:1766098301.727112:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175808:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098301.727174:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417175808:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 61us (1551us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098301.727181:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1214
00010000:00010000:1.0:1766098301.821991:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098302.402866:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098302.402890:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098302.402895:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098302.402906:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2805
00000100:00080000:1.0:1766098302.402910:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098302.402917:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098302.402944:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002f9058ba x1851885432856960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098302.402998:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6eeb4e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432856896:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403088:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098302.403124:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098302.403130:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064afdb67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432857024:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403153:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098302.403184:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f9058ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856960:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403211:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098302.403253:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856896
00000100:00100000:1.0:1766098302.403306:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857024
00000100:00100000:1.0:1766098302.403330:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1779
00000100:00100000:1.0:1766098302.403336:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3b84863 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432856896:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403406:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3b84863 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432856896:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (319us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098302.403414:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1779
00000100:00100000:1.0:1766098302.403433:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1780
00000100:00100000:1.0:1766098302.403436:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000940b152d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432857024:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403529:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000940b152d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432857024:12345-0@lo:400:kworker/u8:2.0 Request processed in 91us (376us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098302.403536:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1780
00000100:00100000:1.0:1766098302.403557:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856960
00000100:00100000:1.0:1766098302.403577:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068
00000100:00100000:1.0:1766098302.403581:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444deb29 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432856960:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766098302.403685:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098302.403721:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444deb29 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432856960:12345-0@lo:8:kworker/u8:2.0 Request processed in 139us (509us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098302.403727:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068
00000100:00080000:1.0:1766098302.403771:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098302.403775:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098302.403779:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f9058ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432856960:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403809:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064afdb67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432857024:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.403819:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6eeb4e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432856896:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098302.404069:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed4dd325 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432856832:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098302.404113:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098302.404145:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098302.404150:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051a248a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432857088:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098302.404175:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098302.404200:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857088
00000100:00100000:3.0:1766098302.404226:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 871
00000100:00100000:3.0:1766098302.404230:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007997fad4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432857088:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098302.404264:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007997fad4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432857088:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (89us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098302.404271:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 871
00010000:00080000:3.0:1766098302.404300:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000051a248a0 x1851885432857088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098309 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098302.404310:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051a248a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432857088:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.406691:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432856832
00000100:00100000:1.0:1766098302.406787:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1215
00000100:00100000:1.0:1766098302.406795:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432856832:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098302.406929:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432856832:12345-0@lo:400:kworker/u8:2.0 Request processed in 135us (2816us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098302.406937:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1215
00000100:00100000:3.0:1766098302.407694:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed4dd325 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432856832:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098302.845842:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098303.869737:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098304.897236:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098305.919065:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098306.844602:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098306.844805:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098306.845636:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176128
00000100:00100000:1.0:1766098306.846788:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176000
00000100:00100000:1.0:1766098306.846850:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1216
00000100:00100000:1.0:1766098306.846858:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417176000:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098306.846954:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417176000:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 96us (2151us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098306.846962:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1216
00000100:00100000:2.0:1766098306.848367:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1069
00000100:00100000:2.0:1766098306.848378:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5762093 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417176128:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:3.0:1766098306.849532:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098306.849677:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176064
00000100:00100000:1.0:1766098306.849744:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1781
00000100:00100000:1.0:1766098306.849751:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fe70e9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417176064:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098306.849854:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fe70e9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417176064:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 103us (329us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098306.849863:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1781
00010000:02020000:2.0:1766098306.852027:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098306.852120:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5762093 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417176128:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 3741us (7530us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098306.852136:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1069
00010000:00010000:0.0:1766098306.942607:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098307.518044:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098307.518053:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098307.518058:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098307.518067:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2810
00000100:00080000:0.0:1766098307.518072:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098307.518080:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098307.518105:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bd34a487 x1851885432857280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098307.518210:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c15dfb05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432857408:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098307.518296:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098307.518348:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098307.518353:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432857152:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098307.518384:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098307.518979:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857408
00000100:00100000:0.0:1766098307.519042:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 872
00000100:00100000:0.0:1766098307.519049:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432857408:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098307.519127:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432857408:12345-0@lo:400:kworker/u8:0.0 Request processed in 77us (831us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098307.519136:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 872
00010000:00080000:2.0:1766098307.519223:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c15dfb05 x1851885432857408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098314 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098307.519238:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c15dfb05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432857408:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098307.519327:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432857344:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098307.519366:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098307.519402:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098307.519408:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432857216:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098307.519430:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098307.519473:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd34a487 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432857280:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098307.519522:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098307.519563:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857152
00000100:00100000:1.0:1766098307.519593:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1217
00000100:00100000:1.0:1766098307.519597:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432857152:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098307.519634:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432857152:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (1249us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098307.519640:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1217
00000100:00100000:1.0:1766098307.519696:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432857152:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098307.519720:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857280
00000100:00100000:1.0:1766098307.519737:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1070
00000100:00100000:1.0:1766098307.519741:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f9058ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432857280:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098307.519841:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098307.519875:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f9058ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432857280:12345-0@lo:8:kworker/u8:0.0 Request processed in 133us (352us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098307.519881:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1070
00000100:00080000:1.0:1766098307.519905:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098307.519909:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098307.519914:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd34a487 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432857280:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098307.519992:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857344
00000100:00100000:2.0:1766098307.520036:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857216
00000100:00100000:2.0:1766098307.520054:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1782
00000100:00100000:2.0:1766098307.520057:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6eeb4e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432857344:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098307.520092:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6eeb4e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432857344:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (726us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098307.520098:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1782
00000100:00100000:2.0:1766098307.520109:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1783
00000100:00100000:2.0:1766098307.520112:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064afdb67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432857216:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098307.520131:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064afdb67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432857216:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (701us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098307.520136:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1783
00000100:00100000:1.0:1766098307.520341:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432857216:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098307.520352:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432857344:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098307.968679:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098308.990017:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098310.015723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098311.038215:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098311.964721:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098311.964772:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098311.964884:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176256
00000100:00100000:1.0:1766098311.964938:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1784
00000100:00100000:1.0:1766098311.964944:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063dd23a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417176256:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098311.965013:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063dd23a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417176256:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 68us (301us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098311.965020:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1784
00000100:00100000:0.0:1766098311.965214:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176192
00000100:00100000:0.0:1766098311.965248:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1218
00000100:00100000:0.0:1766098311.965252:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417176192:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098311.965302:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417176192:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 48us (530us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098311.965309:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1218
00000100:00100000:3.0:1766098311.965697:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098311.965948:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176320
00000100:00100000:0.0:1766098311.965984:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1071
00000100:00100000:0.0:1766098311.965989:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417176320:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098311.966085:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098311.966130:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417176320:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 141us (434us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098311.966137:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1071
00010000:00010000:1.0:1766098312.061757:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098312.637962:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098312.637971:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098312.637974:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098312.637983:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2815
00000100:00080000:3.0:1766098312.637986:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:2.0:1766098312.637989:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766098312.637995:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098312.638019:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432857472:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098312.638020:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b4d81768 x1851885432857600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098312.638127:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098312.638147:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432857536:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.638186:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088efffa5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432857728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098312.638300:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857472
00000100:00100000:3.0:1766098312.638353:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1219
00000100:00100000:3.0:1766098312.638360:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432857472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.638425:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098312.638468:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857728
00000100:00100000:2.0:1766098312.638518:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 873
00000100:00100000:2.0:1766098312.638521:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432857728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.638554:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432857728:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (128us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098312.638560:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 873
00010000:00080000:2.0:1766098312.638598:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000088efffa5 x1851885432857728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098319 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098312.638611:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088efffa5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432857728:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.638664:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432857664:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.638690:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098312.639213:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857664
00000100:00100000:2.0:1766098312.639248:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1785
00000100:00100000:2.0:1766098312.639253:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006deaa1cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432857664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.639308:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006deaa1cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432857664:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (617us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098312.639315:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1785
00000100:00100000:2.0:1766098312.639342:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432857664:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.639373:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432857600:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098312.639421:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098312.639448:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857600
00000100:00100000:2.0:1766098312.639465:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1072
00000100:00100000:2.0:1766098312.639469:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432857600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098312.639612:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098312.639644:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432857600:12345-0@lo:8:kworker/u8:0.0 Request processed in 175us (223us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098312.639652:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1072
00000100:00080000:2.0:1766098312.639675:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098312.639679:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098312.639684:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432857600:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098312.641327:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098312.642368:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432857472:12345-0@lo:400:kworker/u8:0.0 Request processed in 3997us (4233us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098312.642388:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1219
00000100:00100000:0.0:1766098312.643319:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432857472:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.644294:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857536
00000100:00100000:2.0:1766098312.644364:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1786
00000100:00100000:2.0:1766098312.644371:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed4dd325 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432857536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098312.644519:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed4dd325 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432857536:12345-0@lo:400:kworker/u8:0.0 Request processed in 147us (3199us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098312.644529:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1786
00000100:00100000:1.0:1766098312.644623:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432857536:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098313.085700:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098314.111505:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098315.135608:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098316.157690:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098317.094926:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098317.095383:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098317.095558:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098317.095665:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176384
00000100:00100000:1.0:1766098317.095716:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1220
00000100:00100000:1.0:1766098317.095721:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417176384:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098317.095916:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417176384:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 194us (998us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098317.095925:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1220
00000100:00100000:0.0:1766098317.096510:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176512
00000100:00100000:0.0:1766098317.096556:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1073
00000100:00100000:0.0:1766098317.096562:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417176512:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098317.096658:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098317.096714:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417176512:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 152us (1157us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098317.096721:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1073
00000100:00100000:2.0:1766098317.097700:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176448
00000100:00100000:2.0:1766098317.097747:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1787
00000100:00100000:2.0:1766098317.097752:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020194846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417176448:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098317.097813:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020194846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417176448:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 60us (2431us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098317.097820:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1787
00010000:00010000:1.0:1766098317.181721:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098317.757736:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098317.757742:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098317.757749:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098317.757758:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2821
00000100:00080000:0.0:1766098317.757761:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098317.757766:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098317.757784:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001cfc5d76 x1851885432857920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098317.757843:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432858048:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.757904:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098317.757917:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432857984:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.757927:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098317.757932:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021f758f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432857792:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.757954:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098317.757955:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098317.757970:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098317.757975:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000684dc236 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432857856:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.757985:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857792
00000100:00100000:2.0:1766098317.757993:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098317.758011:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001cfc5d76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432857920:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098317.758021:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1221
00000100:00100000:0.0:1766098317.758026:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432857792:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098317.758031:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098317.758054:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857984
00000100:00100000:2.0:1766098317.758079:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857856
00000100:00100000:0.0:1766098317.758083:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432857792:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (128us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098317.758090:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1221
00000100:00100000:2.0:1766098317.758094:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1788
00000100:00100000:2.0:1766098317.758098:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fa0a70e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432857984:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.758121:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1789
00000100:00100000:0.0:1766098317.758125:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abde9dbe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432857856:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098317.758128:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fa0a70e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432857984:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (173us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098317.758134:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1788
00000100:00100000:0.0:1766098317.758148:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abde9dbe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432857856:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (154us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098317.758153:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1789
00000100:00100000:2.0:1766098317.758167:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000684dc236 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432857856:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.758172:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858048
00000100:00100000:2.0:1766098317.758176:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432857984:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.758192:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432857920
00000100:00100000:0.0:1766098317.758336:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 874
00000100:00100000:0.0:1766098317.758340:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2a01029 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432858048:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098317.758369:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2a01029 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432858048:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (466us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098317.758374:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 874
00000100:00100000:0.0:1766098317.758530:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1074
00000100:00100000:0.0:1766098317.758533:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432857920:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098317.758629:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098317.758661:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432857920:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (628us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098317.758667:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1074
00000100:00080000:3.0:1766098317.759807:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098317.759812:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098317.759817:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001cfc5d76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432857920:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098317.759853:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021f758f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432857792:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766098317.759864:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004c0ecbea x1851885432858048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098324 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098317.759873:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432858048:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098318.206856:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098319.230357:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098320.254860:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098321.277820:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098322.208219:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098322.211912:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098322.211991:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098322.212126:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176640
00000100:00100000:1.0:1766098322.212177:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1790
00000100:00100000:1.0:1766098322.212183:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fcfa81e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417176640:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098322.212269:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fcfa81e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417176640:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 85us (364us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098322.212276:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1790
00000100:00100000:2.0:1766098322.212428:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176704
00000100:00100000:2.0:1766098322.212461:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1075
00000100:00100000:2.0:1766098322.212465:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000315ba24d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417176704:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098322.212572:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098322.212605:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000315ba24d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417176704:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 139us (615us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098322.212612:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1075
00000100:00100000:0.0:1766098322.213130:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176576
00000100:00100000:1.0:1766098322.214445:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1222
00000100:00100000:1.0:1766098322.214454:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417176576:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098322.214541:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417176576:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 87us (6329us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098322.214549:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1222
00010000:00010000:0.0:1766098322.302644:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098322.878150:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:1.0:1766098322.878153:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e23f899b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432858176:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766098322.878156:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098322.878176:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098322.878181:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2826
00000100:00080000:2.0:1766098322.878183:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098322.878188:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098322.878240:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e2645ec4 x1851885432858240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098322.878244:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098322.878277:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098322.878282:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4f25f0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432858304:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098322.878306:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098322.878317:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858176
00000100:00100000:1.0:1766098322.878328:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858304
00000100:00100000:3.0:1766098322.878409:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432858368:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098322.878440:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098322.878460:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098322.878465:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432858112:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098322.878518:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098322.878542:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432858240:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098322.878563:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098322.878592:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858368
00000100:00100000:3.0:1766098322.878614:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858240
00000100:00100000:3.0:1766098322.878632:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 875
00000100:00100000:3.0:1766098322.878637:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432858368:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098322.878681:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432858368:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (240us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098322.878688:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 875
00000100:00100000:1.0:1766098322.878697:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1791
00000100:00100000:1.0:1766098322.878701:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051a248a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432858176:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098322.878701:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076
00000100:00100000:3.0:1766098322.878704:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432858240:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098322.878733:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051a248a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432858176:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (494us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098322.878739:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1791
00000100:00100000:1.0:1766098322.878749:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1792
00000100:00100000:1.0:1766098322.878752:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062911c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432858304:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098322.878772:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062911c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432858304:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (467us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098322.878777:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1792
00010000:02020000:3.0:1766098322.878799:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098322.878809:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4f25f0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432858304:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098322.878819:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e23f899b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432858176:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098322.878826:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432858240:12345-0@lo:8:kworker/u8:2.0 Request processed in 122us (264us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098322.878833:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076
00000100:00080000:3.0:1766098322.878857:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098322.878861:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098322.878864:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432858240:0@lo:8:kworker/u8:2.0
00010000:00080000:3.0:1766098322.878897:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bba96c12 x1851885432858368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098329 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098322.878906:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432858368:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098322.879587:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858112
00000100:00100000:0.0:1766098322.879652:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1223
00000100:00100000:0.0:1766098322.879660:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432858112:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098322.879766:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432858112:12345-0@lo:400:kworker/u8:2.0 Request processed in 104us (1245us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098322.879775:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1223
00000100:00100000:0.0:1766098322.879850:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432858112:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098323.327863:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098324.349871:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098325.373685:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098326.397700:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098327.324870:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098327.324935:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098327.325062:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176768
00000100:00100000:2.0:1766098327.325136:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1224
00000100:00100000:2.0:1766098327.325144:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417176768:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098327.325254:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417176768:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 109us (319us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098327.325263:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1224
00000100:00100000:3.0:1766098327.325350:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176832
00000100:00100000:3.0:1766098327.325395:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1793
00000100:00100000:3.0:1766098327.325400:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac54f788 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417176832:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098327.325437:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac54f788 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417176832:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 36us (579us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098327.325444:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1793
00000100:00100000:3.0:1766098327.328967:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098327.329109:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176896
00000100:00100000:2.0:1766098327.329181:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1077
00000100:00100000:2.0:1766098327.329188:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417176896:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098327.329295:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098327.329383:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417176896:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 194us (427us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098327.329392:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1077
00010000:00010000:1.0:1766098327.421863:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098327.999062:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000667f5d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432858432:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098327.999070:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098327.999094:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098327.999099:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098327.999105:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2831
00000100:00080000:1.0:1766098327.999108:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098327.999114:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098327.999139:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a46965fb x1851885432858560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098327.999179:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098327.999216:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098327.999223:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432858688:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999241:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4d39d96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432858496:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098327.999257:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098327.999283:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098327.999310:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098327.999315:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025358442 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432858624:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999336:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098327.999336:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858688
00000100:00100000:1.0:1766098327.999367:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a46965fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432858560:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999396:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098327.999396:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 876
00000100:00100000:2.0:1766098327.999407:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b110edbb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432858688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098327.999473:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b110edbb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432858688:12345-0@lo:400:kworker/u8:0.0 Request processed in 68us (216us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098327.999507:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 876
00000100:00100000:2.0:1766098327.999527:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858560
00000100:00100000:1.0:1766098327.999546:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858496
00000100:00100000:2.0:1766098327.999550:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1078
00000100:00100000:2.0:1766098327.999555:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000022bd8681 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432858560:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999580:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858624
00000100:00100000:1.0:1766098327.999600:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1794
00000100:00100000:1.0:1766098327.999604:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f9058ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432858496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999639:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f9058ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432858496:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (359us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098327.999645:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1794
00010000:02020000:2.0:1766098327.999657:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098327.999659:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1795
00000100:00100000:1.0:1766098327.999664:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063fed1c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432858624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999700:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063fed1c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432858624:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (363us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098327.999706:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000022bd8681 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432858560:12345-0@lo:8:kworker/u8:0.0 Request processed in 149us (312us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098327.999718:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1078
00000100:00100000:1.0:1766098327.999722:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1795
00010000:00080000:2.0:1766098327.999752:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000444deb29 x1851885432858688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098334 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098327.999766:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432858688:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098327.999798:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098327.999803:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098327.999808:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a46965fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432858560:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999835:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000025358442 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432858624:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098327.999847:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4d39d96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432858496:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098328.000385:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858432
00000100:00100000:2.0:1766098328.001589:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1225
00000100:00100000:2.0:1766098328.001602:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432858432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098328.001714:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432858432:12345-0@lo:400:kworker/u8:0.0 Request processed in 113us (2539us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098328.001722:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1225
00000100:00100000:1.0:1766098328.001829:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000667f5d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432858432:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098328.445716:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098329.471431:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098330.494675:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098331.518403:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098332.444435:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098332.444541:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177024
00000100:00100000:3.0:1766098332.444546:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098332.444583:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098332.444619:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1796
00000100:00100000:1.0:1766098332.444628:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7865906 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417177024:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098332.444736:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7865906 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417177024:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 108us (315us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098332.444746:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1796
00000100:00100000:0.0:1766098332.444814:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417176960
00000100:00100000:0.0:1766098332.444865:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1226
00000100:00100000:0.0:1766098332.444870:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417176960:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098332.444899:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417176960:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 28us (356us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098332.444905:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1226
00000100:00100000:2.0:1766098332.445990:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177088
00000100:00100000:2.0:1766098332.446036:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1079
00000100:00100000:2.0:1766098332.446042:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417177088:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098332.446159:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098332.446243:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417177088:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 200us (1659us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098332.446251:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1079
00010000:00010000:1.0:1766098332.541718:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098333.120283:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098333.120294:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098333.120299:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098333.120309:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2836
00000100:00080000:2.0:1766098333.120314:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098333.120322:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098333.120353:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000aad3cd34 x1851885432858880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098333.121004:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8a0c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432859008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098333.121095:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098333.121149:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098333.121156:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea586a1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432858752:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098333.121190:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098333.121191:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000044a514eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432858816:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098333.121253:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad3cd34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432858880:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098333.121281:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098333.121282:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098333.121300:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098333.121305:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1e9dc67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432858944:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098333.121328:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098333.121368:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858816
00000100:00100000:1.0:1766098333.121417:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858944
00000100:00100000:1.0:1766098333.121462:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1797
00000100:00100000:1.0:1766098333.121468:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f424afea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432858816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098333.121554:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f424afea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432858816:12345-0@lo:400:kworker/u8:2.0 Request processed in 84us (271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098333.121562:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1797
00000100:00100000:1.0:1766098333.121580:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1798
00000100:00100000:1.0:1766098333.121584:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8acee97 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432858944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098333.121610:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8acee97 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432858944:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (282us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098333.121616:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1798
00000100:00100000:1.0:1766098333.121671:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1e9dc67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432858944:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098333.121683:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000044a514eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432858816:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098333.121840:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859008
00000100:00100000:2.0:1766098333.121868:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858880
00000100:00100000:2.0:1766098333.121891:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 877
00000100:00100000:2.0:1766098333.121895:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432859008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098333.121938:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432859008:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (846us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098333.121944:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 877
00000100:00100000:2.0:1766098333.121956:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080
00000100:00100000:2.0:1766098333.121959:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432858880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766098333.122059:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:3.0:1766098333.122091:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006af8a0c7 x1851885432859008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098340 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098333.122093:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432858880:12345-0@lo:8:kworker/u8:2.0 Request processed in 133us (813us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098333.122100:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080
00000100:00100000:3.0:1766098333.122104:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8a0c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432859008:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098333.122141:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098333.122156:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098333.122160:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad3cd34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432858880:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098333.123619:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432858752
00000100:00100000:1.0:1766098333.124085:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1227
00000100:00100000:1.0:1766098333.124095:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432858752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098333.127802:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432858752:12345-0@lo:400:kworker/u8:2.0 Request processed in 3694us (6597us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098333.127825:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1227
00000100:00100000:2.0:1766098333.128181:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea586a1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432858752:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098333.566406:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098334.590461:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098335.614273:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098336.639414:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098337.566131:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098337.566188:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098337.566260:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177216
00000100:00100000:1.0:1766098337.566274:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177152
00000100:00100000:1.0:1766098337.566318:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1228
00000100:00100000:1.0:1766098337.566325:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417177152:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098337.566453:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417177152:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 127us (328us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098337.566466:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1228
00000100:00100000:1.0:1766098337.566580:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098337.566649:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177280
00000100:00100000:1.0:1766098337.566667:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1081
00000100:00100000:1.0:1766098337.566672:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417177280:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098337.566766:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098337.566793:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417177280:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 121us (214us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098337.566799:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1081
00000100:00100000:2.0:1766098337.566856:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1799
00000100:00100000:2.0:1766098337.566863:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011bca332 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417177216:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098337.566896:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011bca332 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417177216:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (709us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098337.566902:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1799
00010000:00010000:0.0:1766098337.663248:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098338.239905:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098338.239912:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098338.239916:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098338.239923:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2841
00000100:00080000:1.0:1766098338.239927:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098338.239932:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098338.239952:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432859136:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098338.239954:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000003b38993 x1851885432859200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098338.240000:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432859264:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098338.240021:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098338.240038:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098338.240072:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003b38993 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432859200:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098338.240094:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859136
00000100:00100000:0.0:1766098338.240115:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098338.240136:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859264
00000100:00100000:0.0:1766098338.240160:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859200
00000100:00100000:1.0:1766098338.240161:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1800
00000100:00100000:1.0:1766098338.240167:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd34a487 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432859136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098338.240189:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1082
00000100:00100000:0.0:1766098338.240194:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432859200:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098338.240214:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd34a487 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432859136:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (195us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098338.240234:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1800
00000100:00100000:1.0:1766098338.240249:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1801
00000100:00100000:1.0:1766098338.240252:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000044e85f33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432859264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098338.240285:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000044e85f33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432859264:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (249us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098338.240291:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1801
00000100:00100000:1.0:1766098338.240331:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432859264:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766098338.240343:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098338.240369:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432859136:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098338.240382:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432859200:12345-0@lo:8:kworker/u8:0.0 Request processed in 186us (267us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098338.240390:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1082
00000100:00080000:1.0:1766098338.240409:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098338.240413:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098338.240418:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003b38993 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432859200:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098338.240620:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432859072:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098338.240651:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098338.240676:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098338.240680:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005db784ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432859328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098338.240699:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098338.240720:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859328
00000100:00100000:3.0:1766098338.240739:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 878
00000100:00100000:3.0:1766098338.240742:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432859328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098338.240769:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432859328:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (70us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098338.240775:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 878
00010000:00080000:3.0:1766098338.240794:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005db784ff x1851885432859328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098345 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098338.240802:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005db784ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432859328:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098338.241176:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859072
00000100:00100000:1.0:1766098338.241305:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1229
00000100:00100000:1.0:1766098338.241309:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432859072:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098338.241582:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432859072:12345-0@lo:400:kworker/u8:0.0 Request processed in 271us (930us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098338.241588:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1229
00000100:00100000:3.0:1766098338.241763:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432859072:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098338.685844:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098339.710023:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098340.734904:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098341.760887:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098342.684198:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098342.684390:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177344
00000100:00100000:0.0:1766098342.684456:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1230
00000100:00100000:0.0:1766098342.684463:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417177344:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098342.684725:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417177344:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 261us (538us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098342.684734:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1230
00000100:00100000:0.0:1766098342.688239:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098342.688462:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177408
00000100:00100000:2.0:1766098342.689266:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1802
00000100:00100000:2.0:1766098342.689276:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c15dfb05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417177408:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098342.690730:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c15dfb05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417177408:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1446us (2495us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098342.690745:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1802
00000100:00100000:1.0:1766098342.695629:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098342.696153:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177472
00000100:00100000:2.0:1766098342.696215:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1083
00000100:00100000:2.0:1766098342.696225:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003b38993 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417177472:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098342.696350:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098342.696440:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003b38993 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417177472:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 215us (824us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098342.696449:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1083
00010000:00010000:1.0:1766098342.782360:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098343.360112:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098343.360138:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098343.360142:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098343.360153:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2846
00000100:00080000:1.0:1766098343.360157:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098343.360164:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098343.360191:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008739e800 x1851885432859520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098343.360242:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d230af01 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432859648:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360327:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098343.360364:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098343.360370:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432859392:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360400:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098343.360426:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432859456:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360456:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098343.360495:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098343.360500:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432859584:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360522:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098343.360550:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432859520:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360571:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098343.360611:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859392
00000100:00100000:1.0:1766098343.360664:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1231
00000100:00100000:1.0:1766098343.360671:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432859392:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360737:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432859392:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (336us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098343.360746:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1231
00000100:00100000:1.0:1766098343.360806:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432859392:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360833:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859648
00000100:00100000:1.0:1766098343.360858:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859520
00000100:00100000:1.0:1766098343.360877:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 879
00000100:00100000:1.0:1766098343.360880:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091c0cb74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432859648:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.360913:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091c0cb74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432859648:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (589us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098343.360918:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 879
00000100:00100000:1.0:1766098343.360928:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1084
00000100:00100000:1.0:1766098343.360931:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432859520:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766098343.361038:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098343.361078:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432859520:12345-0@lo:8:kworker/u8:2.0 Request processed in 145us (506us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098343.361084:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1084
00000100:00080000:1.0:1766098343.361115:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098343.361120:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098343.361123:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432859520:0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766098343.361144:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d230af01 x1851885432859648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098350 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766098343.361155:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d230af01 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432859648:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.361181:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859456
00000100:00100000:1.0:1766098343.361206:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859584
00000100:00100000:1.0:1766098343.361222:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1803
00000100:00100000:1.0:1766098343.361225:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001935c591 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432859456:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.361258:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001935c591 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432859456:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (804us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098343.361273:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1803
00000100:00100000:1.0:1766098343.361286:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1804
00000100:00100000:1.0:1766098343.361290:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004bfac224 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432859584:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.361315:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004bfac224 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432859584:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (792us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098343.361320:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1804
00000100:00100000:1.0:1766098343.361351:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432859584:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098343.361360:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432859456:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098343.809291:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098344.835143:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098345.853803:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098346.883827:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098347.809609:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098347.809677:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098347.809766:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177536
00000100:00100000:1.0:1766098347.809794:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177600
00000100:00100000:0.0:1766098347.809836:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1232
00000100:00100000:1.0:1766098347.809836:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1805
00000100:00100000:0.0:1766098347.809844:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417177536:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098347.809844:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088efffa5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417177600:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098347.809940:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417177536:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 96us (263us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098347.809949:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1232
00000100:00100000:1.0:1766098347.810290:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088efffa5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417177600:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 444us (690us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098347.810298:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1805
00000100:00100000:0.0:1766098347.810299:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098347.810685:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177664
00000100:00100000:2.0:1766098347.810722:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1085
00000100:00100000:2.0:1766098347.810728:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c0ecbea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417177664:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098347.810826:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098347.810877:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c0ecbea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417177664:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 148us (578us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098347.810884:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1085
00010000:00010000:1.0:1766098347.902448:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098348.477761:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098348.477768:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098348.477771:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098348.477778:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2851
00000100:00080000:1.0:1766098348.477782:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098348.477787:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098348.477811:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d230af01 x1851885432859840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098348.477852:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432859776:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098348.477903:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432859904:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098348.477961:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098348.477981:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d230af01 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432859840:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098348.478093:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432859968:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098348.478111:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432859712:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098348.481309:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098348.481387:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859840
00000100:00100000:1.0:1766098348.481425:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1086
00000100:00100000:1.0:1766098348.481431:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c29453e3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432859840:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098348.481472:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:02020000:1.0:1766098348.481559:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098348.481577:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859776
00000100:00100000:1.0:1766098348.481623:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c29453e3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432859840:12345-0@lo:8:kworker/u8:2.0 Request processed in 190us (319us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098348.481639:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1806
00000100:00100000:1.0:1766098348.481640:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1086
00000100:00100000:0.0:1766098348.481650:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce3f3b2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432859776:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098348.481696:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098348.481703:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098348.481709:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d230af01 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432859840:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098348.481714:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce3f3b2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432859776:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (3754us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098348.481723:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1806
00000100:00100000:1.0:1766098348.481769:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859712
00000100:00100000:1.0:1766098348.481814:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1233
00000100:00100000:1.0:1766098348.481824:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432859712:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098348.481858:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098348.483128:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859904
00000100:00100000:0.0:1766098348.483182:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1807
00000100:00100000:0.0:1766098348.483188:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021f758f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432859904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098348.483252:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021f758f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432859904:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (1397us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098348.483260:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1807
00000100:00100000:0.0:1766098348.483304:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432859904:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098348.483357:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432859712:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098348.490979:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432859712:12345-0@lo:400:kworker/u8:2.0 Request processed in 9151us (9504us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098348.490995:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1233
00000100:00100000:2.0:1766098348.491387:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098348.491397:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432859776:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098348.491501:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432859968
00000100:00100000:1.0:1766098348.491551:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 880
00000100:00100000:1.0:1766098348.491557:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea586a1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432859968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098348.491648:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea586a1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432859968:12345-0@lo:400:kworker/u8:2.0 Request processed in 90us (267us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098348.491671:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 880
00010000:00080000:3.0:1766098348.491701:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008edde9a8 x1851885432859968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098355 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098348.491731:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432859968:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098348.931902:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098349.950094:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098350.973976:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098351.998204:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098352.932070:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098352.933395:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098352.934086:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177792
00000100:00100000:0.0:1766098352.935400:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098352.935466:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177856
00000100:00100000:0.0:1766098352.935535:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1087
00000100:00100000:0.0:1766098352.935542:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417177856:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098352.935649:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098352.935730:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417177856:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 188us (337us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098352.935738:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1087
00000100:00100000:1.0:1766098352.936021:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1808
00000100:00100000:1.0:1766098352.936029:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001cfc5d76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417177792:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098352.936101:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001cfc5d76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417177792:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 72us (4044us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098352.936108:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1808
00000100:00100000:3.0:1766098352.936177:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177728
00000100:00100000:3.0:1766098352.936312:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1234
00000100:00100000:3.0:1766098352.936317:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417177728:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098352.936347:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417177728:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 30us (2959us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098352.936353:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1234
00010000:00010000:0.0:1766098353.023039:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098353.600681:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098353.600692:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098353.600697:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098353.600709:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2856
00000100:00080000:2.0:1766098353.600714:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098353.600721:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098353.600746:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f18ab927 x1851885432860160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098353.600840:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc410d1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432860288:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098353.600904:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001de8aed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432860096:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098353.600937:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098353.600951:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098353.600999:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098353.601006:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac943495 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432860224:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098353.601018:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860096
00000100:00100000:3.0:1766098353.601034:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098353.601068:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098353.601072:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860224
00000100:00100000:3.0:1766098353.601077:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000948c2759 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432860032:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098353.601099:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1809
00000100:00100000:2.0:1766098353.601107:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb80abc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432860096:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098353.601116:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098353.601145:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860032
00000100:00100000:2.0:1766098353.601174:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb80abc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432860096:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (223us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098353.601184:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1809
00000100:00100000:3.0:1766098353.601187:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1235
00000100:00100000:3.0:1766098353.601193:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432860032:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098353.601207:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1810
00000100:00100000:2.0:1766098353.601212:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba0acca1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432860224:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098353.601228:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432860032:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (113us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098353.601238:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1235
00000100:00100000:2.0:1766098353.601278:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba0acca1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432860224:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (242us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098353.601287:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1810
00000100:00100000:3.0:1766098353.601304:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac943495 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432860224:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098353.601321:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001de8aed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432860096:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098353.601332:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000948c2759 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432860032:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098353.601334:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860288
00000100:00100000:0.0:1766098353.601368:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 881
00000100:00100000:0.0:1766098353.601373:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043193b76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432860288:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098353.601568:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043193b76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432860288:12345-0@lo:400:kworker/u8:2.0 Request processed in 194us (634us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098353.601576:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 881
00000100:00100000:1.0:1766098353.601588:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f18ab927 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432860160:0@lo:8:kworker/u8:2.0
00010000:00080000:3.0:1766098353.601604:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fc410d1b x1851885432860288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098360 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098353.601616:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc410d1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432860288:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098353.601631:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098353.601669:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860160
00000100:00100000:0.0:1766098353.601688:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088
00000100:00100000:0.0:1766098353.601692:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d230af01 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432860160:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098353.601792:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098353.601844:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d230af01 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432860160:12345-0@lo:8:kworker/u8:2.0 Request processed in 150us (216us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098353.601851:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088
00000100:00080000:2.0:1766098353.601930:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098353.601935:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098353.601940:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f18ab927 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432860160:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766098354.047285:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098355.070466:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098356.093833:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098357.118771:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098358.044531:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098358.044717:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177920
00000100:00100000:3.0:1766098358.044789:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1236
00000100:00100000:3.0:1766098358.044797:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417177920:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098358.045100:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417177920:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 302us (579us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098358.045109:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1236
00000100:00100000:0.0:1766098358.045638:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098358.045725:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417177984
00000100:00100000:1.0:1766098358.046151:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1811
00000100:00100000:1.0:1766098358.046158:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000684dc236 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417177984:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098358.046837:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000684dc236 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417177984:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 676us (1274us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098358.046851:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1811
00000100:00100000:1.0:1766098358.048966:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098358.049278:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178048
00000100:00100000:0.0:1766098358.049358:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1089
00000100:00100000:0.0:1766098358.049366:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059cbb111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417178048:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098358.049467:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098358.049815:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059cbb111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417178048:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 448us (859us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098358.049823:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1089
00010000:00010000:1.0:1766098358.144860:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098358.718796:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432860352:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098358.718888:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098358.720405:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432860416:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098358.720473:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098358.720560:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860416
00000100:00100000:1.0:1766098358.720622:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1812
00000100:00100000:1.0:1766098358.720628:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f18ab927 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432860416:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098358.720991:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860352
00000100:00100000:0.0:1766098358.721025:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1237
00000100:00100000:0.0:1766098358.721030:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432860352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098358.721075:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432860352:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (2188us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098358.721082:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1237
00000100:00100000:0.0:1766098358.721167:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432860352:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098358.721356:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098358.721362:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098358.721365:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098358.721370:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2862
00000100:00080000:3.0:1766098358.721372:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098358.721376:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098358.721404:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ff2504ac x1851885432860480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098358.721954:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432860480:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098358.722001:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098358.722108:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f18ab927 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432860416:12345-0@lo:400:kworker/u8:0.0 Request processed in 1478us (1635us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098358.722116:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1812
00000100:00100000:2.0:1766098358.722186:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432860416:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098358.722201:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098358.722206:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f3d8a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432860544:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098358.722230:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098358.722311:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860544
00000100:00100000:1.0:1766098358.722349:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1813
00000100:00100000:1.0:1766098358.722353:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4af4447 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432860544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098358.722388:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4af4447 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432860544:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (159us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098358.722393:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1813
00000100:00100000:2.0:1766098358.722454:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f3d8a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432860544:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098358.722626:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432860608:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098358.722655:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098358.722753:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860480
00000100:00100000:2.0:1766098358.722784:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860608
00000100:00100000:2.0:1766098358.722814:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 882
00000100:00100000:2.0:1766098358.722819:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa723212 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432860608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098358.722855:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa723212 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432860608:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (200us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098358.722875:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 882
00000100:00100000:2.0:1766098358.722889:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1090
00000100:00100000:2.0:1766098358.722892:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000948c2759 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432860480:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098358.722999:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098358.723031:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000948c2759 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432860480:12345-0@lo:8:kworker/u8:0.0 Request processed in 138us (1031us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098358.723037:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1090
00000100:00080000:2.0:1766098358.723060:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098358.723063:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098358.723067:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432860480:0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766098358.723170:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fc68c7d9 x1851885432860608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098365 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098358.723180:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432860608:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098359.167123:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098360.189738:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098361.213691:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098362.238409:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098363.167853:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098363.167935:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098363.168063:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098363.168291:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178240
00000100:00100000:2.0:1766098363.168360:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1091
00000100:00100000:2.0:1766098363.168369:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417178240:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098363.168506:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098363.168620:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417178240:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 250us (555us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098363.168630:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1091
00000100:00100000:3.0:1766098363.168891:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178176
00000100:00100000:3.0:1766098363.168953:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1814
00000100:00100000:3.0:1766098363.168958:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417178176:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098363.169085:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417178176:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 125us (1243us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098363.169092:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1814
00000100:00100000:1.0:1766098363.169253:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178112
00000100:00100000:1.0:1766098363.169297:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1238
00000100:00100000:1.0:1766098363.169303:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417178112:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098363.172828:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417178112:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 3511us (4879us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098363.172849:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1238
00010000:00010000:1.0:1766098363.261796:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098363.841603:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098363.841613:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098363.841618:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098363.841629:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2867
00000100:00080000:0.0:1766098363.841632:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098363.841640:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098363.841666:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000036f32338 x1851885432860800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098363.841746:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036f32338 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432860800:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098363.841829:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098363.841900:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860800
00000100:00100000:0.0:1766098363.841944:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092
00000100:00100000:0.0:1766098363.841951:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432860800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098363.842059:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098363.842101:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432860800:12345-0@lo:8:kworker/u8:0.0 Request processed in 149us (273us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098363.842109:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092
00000100:00080000:0.0:1766098363.842164:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098363.842167:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098363.842172:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036f32338 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432860800:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098363.842221:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432860928:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098363.842246:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098363.842262:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098363.842268:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a150dbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432860672:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098363.842298:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098363.842325:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860928
00000100:00100000:0.0:1766098363.842355:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 883
00000100:00100000:0.0:1766098363.842359:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432860928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098363.842403:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432860928:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (157us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098363.842409:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 883
00010000:00080000:0.0:1766098363.842433:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c5762093 x1851885432860928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098370 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098363.842453:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432860928:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098363.842515:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860672
00000100:00100000:0.0:1766098363.842541:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1239
00000100:00100000:0.0:1766098363.842545:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432860672:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098363.842578:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432860672:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (280us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098363.842585:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1239
00000100:00100000:0.0:1766098363.842616:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a150dbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432860672:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098363.842713:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432860736:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098363.842749:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098363.842780:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098363.842786:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007997fad4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432860864:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098363.842807:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098363.842838:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860736
00000100:00100000:2.0:1766098363.842875:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860864
00000100:00100000:2.0:1766098363.842903:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1815
00000100:00100000:2.0:1766098363.842907:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009557e695 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432860736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098363.842941:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009557e695 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432860736:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (192us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098363.842948:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1815
00000100:00100000:2.0:1766098363.842960:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1816
00000100:00100000:2.0:1766098363.842963:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1649ff3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432860864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098363.842984:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1649ff3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432860864:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (178us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098363.842989:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1816
00000100:00100000:2.0:1766098363.843030:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007997fad4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432860864:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098363.843039:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432860736:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098364.286793:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098365.309737:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098366.333752:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098367.360965:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098368.287239:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098368.287301:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098368.287330:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098368.287352:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178432
00000100:00100000:0.0:1766098368.287416:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178368
00000100:00100000:1.0:1766098368.287418:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1093
00000100:00100000:1.0:1766098368.287426:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a093afa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417178432:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:0.0:1766098368.287461:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1817
00000100:00100000:0.0:1766098368.287471:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003825ca08 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417178368:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00010000:02020000:1.0:1766098368.287570:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098368.287730:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178304
00000100:00100000:3.0:1766098368.287780:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1240
00000100:00100000:3.0:1766098368.287785:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417178304:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.288653:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003825ca08 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417178368:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 1175us (1317us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098368.288653:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a093afa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417178432:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1222us (1347us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098368.288665:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417178304:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 877us (1436us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098368.288667:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1817
00000100:00100000:1.0:1766098368.288667:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1093
00000100:00100000:3.0:1766098368.288677:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1240
00010000:00010000:0.0:1766098368.383662:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098368.959746:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098368.959755:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098368.959758:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098368.959766:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2872
00000100:00080000:2.0:1766098368.959770:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098368.959775:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098368.959797:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ac943495 x1851885432861120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098368.959842:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad3cd34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432861184:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098368.959931:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098368.959962:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098368.959967:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001de8aed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432861056:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098368.959987:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098368.960019:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861184
00000100:00100000:2.0:1766098368.960056:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861056
00000100:00100000:2.0:1766098368.960073:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1818
00000100:00100000:2.0:1766098368.960079:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000044a514eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432861184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098368.960125:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000044a514eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432861184:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (195us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098368.960132:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1818
00000100:00100000:2.0:1766098368.960149:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1819
00000100:00100000:2.0:1766098368.960153:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1e9dc67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432861056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098368.960178:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1e9dc67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432861056:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (191us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098368.960183:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1819
00000100:00100000:2.0:1766098368.960233:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001de8aed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432861056:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098368.960243:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad3cd34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432861184:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960328:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac943495 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432861120:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960360:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098368.960405:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc410d1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432860992:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960432:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098368.960451:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098368.960456:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8a0c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432861248:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960501:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098368.960542:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432860992
00000100:00100000:0.0:1766098368.960569:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1241
00000100:00100000:0.0:1766098368.960573:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432860992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960607:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432860992:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (175us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098368.960613:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1241
00000100:00100000:0.0:1766098368.960642:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861120
00000100:00100000:0.0:1766098368.960656:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861248
00000100:00100000:0.0:1766098368.960673:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 884
00000100:00100000:0.0:1766098368.960676:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432861248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960703:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432861248:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (202us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098368.960710:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 884
00000100:00100000:0.0:1766098368.960720:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1094
00000100:00100000:0.0:1766098368.960724:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5762093 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432861120:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098368.960813:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098368.960843:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5762093 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432861120:12345-0@lo:8:kworker/u8:2.0 Request processed in 118us (482us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098368.960849:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1094
00000100:00080000:0.0:1766098368.960885:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098368.960889:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098368.960893:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac943495 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432861120:0@lo:8:kworker/u8:2.0
00010000:00080000:0.0:1766098368.960913:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006af8a0c7 x1851885432861248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098375 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098368.960921:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8a0c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432861248:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098368.960937:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc410d1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432860992:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098369.405698:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098370.431445:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098371.453933:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098372.478319:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098373.406690:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098373.407436:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098373.407570:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178560
00000100:00100000:0.0:1766098373.407631:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1820
00000100:00100000:0.0:1766098373.407640:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad3cd34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417178560:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098373.407716:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad3cd34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417178560:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 77us (1035us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098373.407723:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178624
00000100:00100000:0.0:1766098373.407726:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1820
00000100:00100000:0.0:1766098373.407873:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1095
00000100:00100000:0.0:1766098373.407879:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001de8aed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417178624:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098373.407975:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098373.408006:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001de8aed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417178624:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 126us (572us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098373.408012:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1095
00000100:00100000:2.0:1766098373.409331:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098373.409449:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178496
00000100:00100000:3.0:1766098373.409532:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1242
00000100:00100000:3.0:1766098373.409540:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417178496:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098373.409626:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417178496:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 86us (303us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098373.409636:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1242
00010000:00010000:1.0:1766098373.503008:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098374.077705:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098374.077715:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098374.077718:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098374.077728:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2877
00000100:00080000:2.0:1766098374.077752:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098374.077758:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098374.077783:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e2645ec4 x1851885432861440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098374.077845:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4f25f0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432861568:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.077935:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098374.077976:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098374.077980:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432861312:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078007:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098374.078028:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e23f899b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432861504:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078050:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098374.078062:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098374.078066:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432861376:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078082:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098374.078119:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861568
00000100:00100000:2.0:1766098374.078171:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 885
00000100:00100000:2.0:1766098374.078178:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432861568:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078235:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432861568:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (301us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098374.078243:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 885
00010000:00080000:2.0:1766098374.078294:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e4f25f0e x1851885432861568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098381 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098374.078307:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4f25f0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432861568:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078350:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432861440:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078373:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098374.078410:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861312
00000100:00100000:2.0:1766098374.078433:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1243
00000100:00100000:2.0:1766098374.078437:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432861312:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078468:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432861312:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (462us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098374.078474:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1243
00000100:00100000:2.0:1766098374.078528:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861440
00000100:00100000:2.0:1766098374.078545:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1096
00000100:00100000:2.0:1766098374.078549:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4f25f0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432861440:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098374.078570:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861504
00000100:00100000:1.0:1766098374.078605:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861376
00000100:00100000:1.0:1766098374.078623:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1821
00000100:00100000:1.0:1766098374.078627:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc410d1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432861504:12345-0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766098374.078647:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098374.078675:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc410d1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432861504:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (623us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098374.078682:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1821
00000100:00100000:2.0:1766098374.078695:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4f25f0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432861440:12345-0@lo:8:kworker/u8:0.0 Request processed in 143us (320us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098374.078701:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1822
00000100:00100000:1.0:1766098374.078710:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006af8a0c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432861376:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078711:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1096
00000100:00080000:2.0:1766098374.078741:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:1.0:1766098374.078745:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006af8a0c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432861376:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (662us total) trans 0 rc 0/0
00000100:00080000:2.0:1766098374.078745:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098374.078751:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432861440:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098374.078767:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1822
00000100:00100000:2.0:1766098374.078774:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432861312:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078809:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432861376:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098374.078827:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e23f899b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432861504:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098374.525689:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098375.550421:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098376.577129:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098377.599771:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098378.525552:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098378.525595:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098378.525671:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098378.525765:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178816
00000100:00100000:1.0:1766098378.525833:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1097
00000100:00100000:1.0:1766098378.525839:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417178816:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098378.525935:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098378.525989:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417178816:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 149us (318us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098378.525996:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1097
00000100:00100000:0.0:1766098378.526072:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178688
00000100:00100000:0.0:1766098378.526111:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1244
00000100:00100000:0.0:1766098378.526115:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417178688:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098378.526147:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417178688:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 32us (553us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098378.526153:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1244
00000100:00100000:3.0:1766098378.526363:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178752
00000100:00100000:3.0:1766098378.526456:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1823
00000100:00100000:3.0:1766098378.526461:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e23f899b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417178752:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098378.526658:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e23f899b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417178752:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 196us (1114us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098378.526666:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1823
00010000:00010000:1.0:1766098378.629349:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098379.199505:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098379.200685:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432861632:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.200786:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098379.201114:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861632
00000100:00100000:1.0:1766098379.201162:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1245
00000100:00100000:1.0:1766098379.201167:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432861632:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098379.201215:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432861632:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (431us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098379.201223:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1245
00000100:00100000:0.0:1766098379.201308:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432861632:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.201493:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:2.0:1766098379.202111:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098379.202117:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098379.202131:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098379.202136:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2882
00000100:00080000:2.0:1766098379.202138:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098379.202142:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098379.202163:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ac943495 x1851885432861760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098379.202212:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac943495 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432861760:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098379.202251:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098379.202310:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861760
00000100:00100000:1.0:1766098379.202333:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1098
00000100:00100000:1.0:1766098379.202336:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000524c5b83 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432861760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766098379.202429:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098379.202585:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432861824:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.202615:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098379.202630:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432861696:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.202646:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098379.202671:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861824
00000100:00100000:0.0:1766098379.202695:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861696
00000100:00100000:0.0:1766098379.202709:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1824
00000100:00100000:0.0:1766098379.202713:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007997fad4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432861824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.202745:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007997fad4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432861824:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (131us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098379.202751:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1824
00000100:00100000:0.0:1766098379.202764:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1825
00000100:00100000:0.0:1766098379.202767:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432861696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.202799:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432861696:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (153us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098379.202805:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1825
00000100:00100000:0.0:1766098379.202843:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a9ccd09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432861824:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098379.202851:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432861696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098379.203102:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098379.203109:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432861888:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098379.203134:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098379.203156:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861888
00000100:00100000:2.0:1766098379.203177:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 886
00000100:00100000:2.0:1766098379.203180:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a9ccd09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432861888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098379.203206:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a9ccd09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432861888:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (72us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098379.203212:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 886
00010000:00080000:2.0:1766098379.203233:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e2645ec4 x1851885432861888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098386 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098379.203242:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2645ec4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432861888:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098379.203709:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000524c5b83 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432861760:12345-0@lo:8:kworker/u8:2.0 Request processed in 1369us (1456us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098379.203718:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1098
00000100:00080000:3.0:1766098379.203813:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098379.203819:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098379.203824:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac943495 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432861760:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766098379.645791:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098380.676057:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098381.694361:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098382.717694:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098383.646621:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098383.648162:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178880
00000100:00100000:0.0:1766098383.648233:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1246
00000100:00100000:0.0:1766098383.648255:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417178880:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098383.648347:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417178880:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 93us (1734us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098383.648354:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1246
00000100:00100000:2.0:1766098383.649024:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098383.650319:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098383.650989:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417178944
00000100:00100000:0.0:1766098383.651071:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1826
00000100:00100000:0.0:1766098383.651077:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2645ec4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417178944:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098383.651169:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2645ec4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417178944:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 91us (2147us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098383.651177:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1826
00000100:00100000:1.0:1766098383.652597:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179008
00000100:00100000:1.0:1766098383.652655:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1099
00000100:00100000:1.0:1766098383.652681:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417179008:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098383.652803:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098383.652868:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417179008:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 187us (2557us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098383.652875:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1099
00010000:00010000:0.0:1766098383.742250:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098384.328941:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098384.328948:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098384.328963:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098384.328972:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2887
00000100:00080000:3.0:1766098384.328976:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098384.328982:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098384.329006:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000093f3d8a9 x1851885432862080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098384.329011:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098384.329045:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432861952:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098384.329072:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432862016:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098384.329309:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098384.329323:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098384.329371:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098384.329377:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432862144:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098384.329398:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098384.329425:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f3d8a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432862080:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098384.329448:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098384.329515:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862016
00000100:00100000:3.0:1766098384.329565:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862144
00000100:00100000:2.0:1766098384.329583:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432862208:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098384.329594:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1827
00000100:00100000:3.0:1766098384.329601:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000394ecbcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432862016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098384.329619:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098384.329675:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000394ecbcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432862016:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (353us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098384.329685:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1827
00000100:00100000:2.0:1766098384.329701:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862080
00000100:00100000:3.0:1766098384.329709:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1828
00000100:00100000:3.0:1766098384.329714:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432862144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098384.329734:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862208
00000100:00100000:3.0:1766098384.329758:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432862144:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (358us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098384.329766:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1828
00000100:00100000:2.0:1766098384.329791:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 887
00000100:00100000:2.0:1766098384.329798:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac943495 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432862208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098384.329816:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100
00000100:00100000:3.0:1766098384.329831:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432862080:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098384.329851:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac943495 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432862208:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (232us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098384.329859:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 887
00010000:00080000:2.0:1766098384.329902:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fe6e47c6 x1851885432862208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098391 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098384.329915:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432862208:0@lo:400:kworker/u8:2.0
00010000:02020000:3.0:1766098384.329923:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098384.329974:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432862080:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (525us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098384.329982:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100
00000100:00100000:2.0:1766098384.330008:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432862144:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098384.330032:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098384.330036:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098384.330041:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f3d8a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432862080:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098384.330712:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432862016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098384.330911:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432861952
00000100:00100000:3.0:1766098384.331682:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1247
00000100:00100000:3.0:1766098384.331690:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432861952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098384.331783:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432861952:12345-0@lo:400:kworker/u8:2.0 Request processed in 93us (2478us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098384.331792:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1247
00000100:00100000:2.0:1766098384.331903:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432861952:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098384.765812:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098385.789780:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098386.814288:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098387.838917:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098388.765674:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098388.765755:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098388.765905:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179072
00000100:00100000:2.0:1766098388.765970:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1248
00000100:00100000:2.0:1766098388.765978:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417179072:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098388.766073:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417179072:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 95us (410us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098388.766081:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1248
00000100:00100000:0.0:1766098388.766173:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179136
00000100:00100000:0.0:1766098388.766212:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1829
00000100:00100000:0.0:1766098388.766217:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d271259f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417179136:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098388.766270:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d271259f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417179136:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 52us (515us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098388.766277:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1829
00000100:00100000:3.0:1766098388.768923:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098388.769454:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179200
00000100:00100000:0.0:1766098388.769530:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1101
00000100:00100000:0.0:1766098388.769536:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041ad3224 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417179200:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098388.769664:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098388.774141:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041ad3224 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417179200:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 4591us (5215us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098388.774187:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1101
00010000:00010000:1.0:1766098388.862444:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098389.439138:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098389.439148:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098389.439152:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098389.439163:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2892
00000100:00080000:1.0:1766098389.439167:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098389.439174:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098389.439201:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000003b38993 x1851885432862400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098389.439260:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091c0cb74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432862464:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098389.439339:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098389.439375:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098389.439381:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432862336:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098389.439405:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098389.439463:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862464
00000100:00100000:1.0:1766098389.439534:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862336
00000100:00100000:1.0:1766098389.439557:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1830
00000100:00100000:1.0:1766098389.439579:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c29453e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432862464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098389.439634:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c29453e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432862464:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (296us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098389.439655:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1830
00000100:00100000:1.0:1766098389.439672:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1831
00000100:00100000:1.0:1766098389.439676:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d230af01 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432862336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.439800:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000022bd8681 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432862272:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.439838:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098389.439861:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098389.439867:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000059a3ed9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432862528:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.439892:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098389.439920:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862528
00000100:00100000:0.0:1766098389.439948:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 888
00000100:00100000:0.0:1766098389.439952:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432862528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.439986:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432862528:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (94us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098389.439993:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 888
00010000:00080000:0.0:1766098389.440033:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000059a3ed9e x1851885432862528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098396 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098389.440047:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000059a3ed9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432862528:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.440073:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862272
00000100:00100000:0.0:1766098389.440097:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1249
00000100:00100000:0.0:1766098389.440101:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432862272:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.440132:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432862272:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (294us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098389.440138:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1249
00000100:00100000:0.0:1766098389.440170:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000022bd8681 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432862272:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.440215:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091c0cb74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432862464:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098389.440251:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003b38993 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432862400:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098389.440278:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098389.440302:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862400
00000100:00100000:0.0:1766098389.440320:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1102
00000100:00100000:0.0:1766098389.440323:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432862400:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098389.440416:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098389.440448:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432862400:12345-0@lo:8:kworker/u8:0.0 Request processed in 124us (169us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098389.440454:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1102
00000100:00080000:0.0:1766098389.440475:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098389.440502:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098389.440506:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003b38993 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432862400:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098389.444551:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432862336:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098389.446253:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d230af01 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432862336:12345-0@lo:400:kworker/u8:0.0 Request processed in 6568us (6841us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098389.446269:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1831
00010000:00010000:0.0:1766098389.890005:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098390.913938:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098391.934181:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098392.957893:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098393.889087:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098393.889147:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098393.889195:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098393.889315:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179328
00000100:00100000:2.0:1766098393.889391:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1832
00000100:00100000:2.0:1766098393.889400:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa723212 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417179328:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098393.889552:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa723212 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417179328:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 151us (405us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098393.889562:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1832
00000100:00100000:3.0:1766098393.889691:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179264
00000100:00100000:3.0:1766098393.889733:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1250
00000100:00100000:3.0:1766098393.889738:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417179264:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098393.889778:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417179264:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 39us (583us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098393.889785:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1250
00000100:00100000:0.0:1766098393.890647:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179392
00000100:00100000:0.0:1766098393.890713:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1103
00000100:00100000:0.0:1766098393.890719:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059cbb111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417179392:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098393.890829:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098393.890899:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059cbb111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417179392:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 179us (1825us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098393.890906:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1103
00010000:00010000:1.0:1766098393.981799:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098394.560876:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098394.560883:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:0.0:1766098394.560888:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098394.560893:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766098394.560911:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432862656:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766098394.560921:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2897
00000100:00080000:0.0:1766098394.560925:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098394.560932:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098394.560961:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ecb38511 x1851885432862720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098394.560995:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098394.561006:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432862784:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098394.561015:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432862720:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098394.561017:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098394.561091:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432862848:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098394.562338:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098394.562436:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862656
00000100:00100000:0.0:1766098394.562521:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1833
00000100:00100000:0.0:1766098394.562529:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b110edbb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432862656:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098394.562600:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b110edbb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432862656:12345-0@lo:400:kworker/u8:2.0 Request processed in 70us (1588us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098394.562607:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1833
00000100:00100000:1.0:1766098394.562680:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098394.562717:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862784
00000100:00100000:0.0:1766098394.562743:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862720
00000100:00100000:1.0:1766098394.562744:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1834
00000100:00100000:1.0:1766098394.562748:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000619fbdfb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432862784:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098394.562761:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104
00000100:00100000:0.0:1766098394.562765:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432862720:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098394.562779:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000619fbdfb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432862784:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (101us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098394.562785:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1834
00000100:00100000:1.0:1766098394.562823:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432862784:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098394.562861:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432862592:0@lo:400:kworker/u8:2.0
00010000:02020000:0.0:1766098394.562863:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098394.562882:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432862656:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098394.562904:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432862720:12345-0@lo:8:kworker/u8:2.0 Request processed in 137us (571us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098394.562912:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104
00000100:00100000:1.0:1766098394.562913:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766098394.562946:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:1.0:1766098394.562947:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862592
00000100:00080000:0.0:1766098394.562950:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098394.562954:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432862720:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098394.562971:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1251
00000100:00100000:1.0:1766098394.562975:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432862592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098394.563006:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432862592:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (93us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098394.563012:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1251
00000100:00100000:0.0:1766098394.563031:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432862592:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098394.563124:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098394.563211:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862848
00000100:00100000:0.0:1766098394.563247:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 889
00000100:00100000:0.0:1766098394.563251:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432862848:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098394.563297:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432862848:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (174us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098394.563303:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 889
00010000:00080000:3.0:1766098394.563370:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002cc1067b x1851885432862848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098401 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098394.563381:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432862848:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098395.006102:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098396.029794:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098397.053925:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098398.079900:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098399.004737:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098399.005717:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098399.005752:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098399.005873:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179584
00000100:00100000:0.0:1766098399.005929:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1105
00000100:00100000:0.0:1766098399.005936:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ccf06dd7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417179584:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:2.0:1766098399.007697:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179456
00000100:00100000:2.0:1766098399.007791:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1252
00000100:00100000:2.0:1766098399.007799:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417179456:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098399.007905:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417179456:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 105us (3181us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098399.007914:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1252
00010000:02020000:0.0:1766098399.009743:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098399.009829:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ccf06dd7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417179584:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 3891us (4117us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098399.009842:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1105
00000100:00100000:3.0:1766098399.011352:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179520
00000100:00100000:3.0:1766098399.011455:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1835
00000100:00100000:3.0:1766098399.011463:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006473b116 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417179520:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098399.011573:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006473b116 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417179520:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 111us (5821us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098399.011581:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1835
00010000:00010000:0.0:1766098399.102498:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098399.677755:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098399.677764:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098399.677768:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098399.677778:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2903
00000100:00080000:0.0:1766098399.677782:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098399.677788:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098399.677812:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000afdf3ced x1851885432863040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098399.677876:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432863040:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098399.677955:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098399.678047:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863040
00000100:00100000:0.0:1766098399.678083:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1106
00000100:00100000:0.0:1766098399.678090:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432863040:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098399.678198:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098399.678259:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432863040:12345-0@lo:8:kworker/u8:2.0 Request processed in 168us (304us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098399.678265:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1106
00000100:00080000:0.0:1766098399.678317:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098399.678321:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098399.678326:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432863040:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098399.678423:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432863168:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098399.678455:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098399.678502:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098399.678507:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432862912:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098399.678533:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098399.678991:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863168
00000100:00100000:0.0:1766098399.679068:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 890
00000100:00100000:0.0:1766098399.679072:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432863168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098399.679323:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432863168:12345-0@lo:400:kworker/u8:2.0 Request processed in 249us (867us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098399.679331:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 890
00000100:00100000:2.0:1766098399.679446:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432863104:0@lo:400:kworker/u8:2.0
00010000:00080000:1.0:1766098399.679474:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009921d012 x1851885432863168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098406 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766098399.679518:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432863168:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098399.679560:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098399.679575:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862912
00000100:00100000:2.0:1766098399.679609:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098399.679618:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1253
00000100:00100000:2.0:1766098399.679640:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432862976:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098399.679641:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432862912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098399.679699:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098399.679709:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432862912:12345-0@lo:400:kworker/u8:2.0 Request processed in 83us (1173us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098399.679719:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1253
00000100:00100000:1.0:1766098399.679775:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432862912:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098399.679818:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863104
00000100:00100000:1.0:1766098399.679851:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432862976
00000100:00100000:1.0:1766098399.679892:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1836
00000100:00100000:1.0:1766098399.679896:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea586a1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432863104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098399.679947:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea586a1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432863104:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (390us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098399.679955:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1836
00000100:00100000:1.0:1766098399.679975:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1837
00000100:00100000:1.0:1766098399.679983:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043193b76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432862976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098399.680009:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043193b76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432862976:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (312us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098399.680015:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1837
00000100:00100000:1.0:1766098399.680069:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432862976:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098399.680081:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432863104:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098400.125983:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098401.150019:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098402.177720:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098403.197774:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098404.126004:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098404.126234:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098404.126331:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179648
00000100:00100000:1.0:1766098404.126409:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1254
00000100:00100000:1.0:1766098404.126416:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417179648:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098404.126472:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179712
00000100:00100000:3.0:1766098404.126557:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1838
00000100:00100000:1.0:1766098404.126564:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417179648:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 145us (329us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098404.126565:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000948c2759 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417179712:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098404.126579:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1254
00000100:00100000:3.0:1766098404.126624:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000948c2759 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417179712:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 60us (633us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098404.126632:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1838
00000100:00100000:1.0:1766098404.130425:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098404.130719:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179776
00000100:00100000:2.0:1766098404.130785:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1107
00000100:00100000:2.0:1766098404.130793:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f827ef16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417179776:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098404.130911:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098404.131022:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f827ef16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417179776:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 227us (606us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098404.131030:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1107
00010000:00010000:1.0:1766098404.222232:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098404.798668:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098404.798679:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098404.798684:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1766098404.798692:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432863232:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098404.798748:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432863296:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098404.798791:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098404.798808:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098404.798948:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863232
00000100:00100000:1.0:1766098404.799003:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1255
00000100:00100000:1.0:1766098404.799010:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432863232:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098404.799048:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863296
00000100:00100000:1.0:1766098404.799061:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432863232:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (273us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098404.799068:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1255
00000100:00100000:2.0:1766098404.799078:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1839
00000100:00100000:2.0:1766098404.799081:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6325418 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432863296:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098404.799126:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6325418 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432863296:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (318us total) trans 0 rc 0/0
00000100:00080000:0.0:1766098404.799134:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2908
00000100:00100000:2.0:1766098404.799136:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1839
00000100:00080000:0.0:1766098404.799139:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098404.799146:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098404.799171:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432863296:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098404.799171:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432863232:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098404.799174:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ecb38511 x1851885432863360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098404.799207:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432863360:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098404.799225:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432863424:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098404.799241:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098404.799249:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432863488:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098404.799251:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098404.799272:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863424
00000100:00100000:1.0:1766098404.799292:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863360
00000100:00100000:2.0:1766098404.799299:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1840
00000100:00100000:0.0:1766098404.799300:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098404.799306:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a093afa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432863424:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098404.799335:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a093afa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432863424:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (86us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098404.799341:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1840
00000100:00100000:2.0:1766098404.799364:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432863424:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098404.800288:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863488
00000100:00100000:3.0:1766098404.800304:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1108
00000100:00100000:3.0:1766098404.800312:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432863360:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098404.800331:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 891
00000100:00100000:1.0:1766098404.800336:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432863488:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098404.800400:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432863488:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (1106us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098404.800408:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 891
00010000:00080000:0.0:1766098404.800419:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009921d012 x1851885432863488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098411 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00010000:02020000:3.0:1766098404.800421:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098404.800434:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009921d012 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432863488:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098404.800501:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432863360:12345-0@lo:8:kworker/u8:2.0 Request processed in 189us (1260us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098404.800508:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1108
00000100:00080000:1.0:1766098404.800521:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098404.800526:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098404.800531:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432863360:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766098405.246257:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098406.269709:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098407.294190:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098408.319314:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098409.247544:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098409.247649:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098409.247682:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098409.247867:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179904
00000100:00100000:1.0:1766098409.247967:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1841
00000100:00100000:1.0:1766098409.247975:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009921d012 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417179904:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098409.248080:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009921d012 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417179904:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 104us (548us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098409.248109:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1841
00000100:00100000:3.0:1766098409.248325:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179968
00000100:00100000:2.0:1766098409.248332:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417179840
00000100:00100000:3.0:1766098409.248357:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1109
00000100:00100000:3.0:1766098409.248363:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417179968:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:2.0:1766098409.248370:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1256
00000100:00100000:2.0:1766098409.248376:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417179840:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098409.248423:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417179840:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 46us (775us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098409.248440:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1256
00010000:02020000:3.0:1766098409.248460:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098409.248535:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417179968:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 170us (851us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098409.248543:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1109
00010000:00010000:1.0:1766098409.344086:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098409.917739:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432863552:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098409.917842:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098409.917907:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863552
00000100:00100000:1.0:1766098409.917967:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1257
00000100:00100000:1.0:1766098409.917974:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432863552:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098409.918040:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432863552:12345-0@lo:400:kworker/u8:0.0 Request processed in 65us (199us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098409.918049:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1257
00000100:00100000:1.0:1766098409.918113:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432863552:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098409.918554:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098409.918560:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098409.918564:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098409.918569:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2913
00000100:00080000:0.0:1766098409.918572:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098409.918576:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098409.918602:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ecb38511 x1851885432863680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098409.918659:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098409.918667:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432863808:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098409.918702:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098409.918733:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863808
00000100:00100000:0.0:1766098409.918762:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 892
00000100:00100000:0.0:1766098409.918766:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000377acd23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432863808:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098409.918808:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000377acd23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432863808:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (106us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098409.918814:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 892
00000100:00100000:1.0:1766098409.918827:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432863616:0@lo:400:kworker/u8:0.0
00010000:00080000:0.0:1766098409.918866:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003cd96141 x1851885432863808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098416 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098409.918875:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098409.918883:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432863808:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098409.918893:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432863680:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098409.918906:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098409.918915:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863616
00000100:00100000:1.0:1766098409.918915:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b413e52a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432863744:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098409.918936:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098409.918953:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1842
00000100:00100000:1.0:1766098409.918956:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098409.918959:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003b38993 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432863616:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098409.918993:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863680
00000100:00100000:0.0:1766098409.918996:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003b38993 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432863616:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (126us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098409.919003:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1842
00000100:00100000:0.0:1766098409.919017:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863744
00000100:00100000:3.0:1766098409.919022:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1110
00000100:00100000:3.0:1766098409.919029:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432863680:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098409.919041:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1843
00000100:00100000:0.0:1766098409.919045:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091c0cb74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432863744:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098409.919068:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091c0cb74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432863744:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (119us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098409.919073:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1843
00010000:02020000:3.0:1766098409.919128:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098409.919146:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b413e52a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432863744:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098409.919160:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432863616:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098409.919186:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432863680:12345-0@lo:8:kworker/u8:0.0 Request processed in 156us (251us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098409.919195:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1110
00000100:00080000:0.0:1766098409.920523:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098409.920534:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098409.920540:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecb38511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432863680:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766098410.367240:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098411.389846:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098412.415170:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098413.440649:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098414.366658:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098414.366740:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098414.366752:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180096
00000100:00100000:3.0:1766098414.366811:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180032
00000100:00100000:2.0:1766098414.366830:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1844
00000100:00100000:2.0:1766098414.366839:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000022bd8681 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417180096:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098414.366866:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1258
00000100:00100000:3.0:1766098414.366874:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417180032:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098414.366949:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000022bd8681 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417180096:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 110us (302us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098414.366958:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1844
00000100:00100000:3.0:1766098414.367135:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417180032:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 252us (386us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098414.367177:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1258
00000100:00100000:1.0:1766098414.371330:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098414.372255:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180160
00000100:00100000:0.0:1766098414.373069:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1111
00000100:00100000:0.0:1766098414.373080:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417180160:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098414.373187:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098414.373350:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417180160:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 269us (2031us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098414.373373:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1111
00010000:00010000:0.0:1766098414.462411:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098415.038581:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098415.038589:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098415.038593:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766098415.038594:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432863872:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098415.038603:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2918
00000100:00080000:1.0:1766098415.038607:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098415.038614:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098415.038640:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004c21f3e8 x1851885432864000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098415.038687:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432863936:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098415.038694:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098415.038723:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432864128:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098415.039066:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864000:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098415.041642:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098415.041689:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098415.041704:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432864064:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098415.041753:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098415.041760:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863872
00000100:00100000:1.0:1766098415.041789:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432863936
00000100:00100000:2.0:1766098415.041825:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1259
00000100:00100000:1.0:1766098415.041829:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864064
00000100:00100000:2.0:1766098415.041834:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432863872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098415.041854:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1845
00000100:00100000:1.0:1766098415.041859:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005db784ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432863936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098415.041918:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432863872:12345-0@lo:400:kworker/u8:2.0 Request processed in 84us (3228us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098415.041927:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1259
00000100:00100000:2.0:1766098415.041987:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008edde9a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432863872:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098415.042009:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005db784ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432863936:12345-0@lo:400:kworker/u8:2.0 Request processed in 138us (365us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098415.042028:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1845
00000100:00100000:0.0:1766098415.042067:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098415.042073:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1846
00000100:00100000:1.0:1766098415.042081:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecb38511 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432864064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098415.042158:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecb38511 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432864064:12345-0@lo:400:kworker/u8:2.0 Request processed in 77us (405us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098415.042166:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1846
00000100:00100000:3.0:1766098415.042227:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098415.042233:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432864064:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098415.042253:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864000
00000100:00100000:1.0:1766098415.042283:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432863936:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098415.042290:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864128
00000100:00100000:3.0:1766098415.042294:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112
00000100:00100000:3.0:1766098415.042301:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432864000:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098415.042318:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 893
00000100:00100000:0.0:1766098415.042323:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432864128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098415.042375:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432864128:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (152us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098415.042383:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 893
00010000:02020000:3.0:1766098415.042408:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098415.042463:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432864000:12345-0@lo:8:kworker/u8:2.0 Request processed in 162us (399us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098415.042469:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112
00010000:00080000:3.0:1766098415.042541:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000047fe747c x1851885432864128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098422 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098415.042553:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432864128:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766098415.043637:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098415.043645:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098415.043650:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864000:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766098415.486129:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098416.509830:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098417.534541:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098418.558267:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098419.492753:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098419.492803:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098419.492835:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180224
00000100:00100000:1.0:1766098419.492895:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1260
00000100:00100000:1.0:1766098419.492902:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417180224:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098419.492905:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098419.492940:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180352
00000100:00100000:2.0:1766098419.492966:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1113
00000100:00100000:2.0:1766098419.492971:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417180352:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:1.0:1766098419.492976:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417180224:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 73us (229us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098419.492984:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1260
00010000:02020000:2.0:1766098419.493065:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098419.493122:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417180352:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 149us (217us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098419.493130:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1113
00000100:00100000:0.0:1766098419.493398:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180288
00000100:00100000:0.0:1766098419.493442:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1847
00000100:00100000:0.0:1766098419.493447:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001de8aed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417180288:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098419.493549:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001de8aed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417180288:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 100us (746us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098419.493555:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1847
00010000:00010000:1.0:1766098419.581689:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098420.158033:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098420.158039:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:1.0:1766098420.158040:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098420.158044:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098420.158053:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2923
00000100:00100000:2.0:1766098420.158055:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000315ba24d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432864256:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098420.158057:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098420.158065:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098420.158085:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432864192:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098420.158089:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ed5b60a4 x1851885432864320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098420.158120:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098420.158134:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098420.158140:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098420.158145:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432864384:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098420.158745:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098420.158749:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864256
00000100:00100000:3.0:1766098420.158787:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098420.158796:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432864448:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098420.158808:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864384
00000100:00100000:1.0:1766098420.158820:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864320:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098420.158831:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098420.158839:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1848
00000100:00100000:2.0:1766098420.158846:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4f25f0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432864256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098420.158855:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098420.158871:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864192
00000100:00100000:2.0:1766098420.158908:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4f25f0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432864256:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (786us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098420.158909:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1849
00000100:00100000:3.0:1766098420.158910:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1261
00000100:00100000:3.0:1766098420.158919:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432864192:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098420.158931:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432864384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098420.158960:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432864192:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (821us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098420.158968:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1261
00000100:00100000:1.0:1766098420.158978:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432864384:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (236us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098420.159009:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864448
00000100:00100000:3.0:1766098420.159031:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864320
00000100:00100000:3.0:1766098420.159048:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 894
00000100:00100000:3.0:1766098420.159052:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432864448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098420.159079:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1114
00000100:00100000:0.0:1766098420.159085:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432864320:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098420.159089:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432864448:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (258us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098420.159096:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 894
00010000:00080000:3.0:1766098420.159130:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e26029d4 x1851885432864448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098427 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098420.159141:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432864448:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098420.159150:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432864192:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098420.159205:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000315ba24d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432864256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098420.159314:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1848
00000100:00100000:2.0:1766098420.159354:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432864384:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098420.159541:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1849
00010000:02020000:0.0:1766098420.159658:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098420.159708:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432864320:12345-0@lo:8:kworker/u8:2.0 Request processed in 623us (856us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098420.159716:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1114
00000100:00080000:1.0:1766098420.159964:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098420.159969:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098420.159973:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864320:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766098420.607093:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098421.630931:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098422.653665:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098423.679369:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098424.604432:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098424.604540:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098424.604555:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180416
00000100:00100000:0.0:1766098424.604611:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1262
00000100:00100000:0.0:1766098424.604617:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417180416:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098424.604648:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180480
00000100:00100000:0.0:1766098424.604691:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417180416:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 72us (266us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098424.604697:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1850
00000100:00100000:0.0:1766098424.604701:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1262
00000100:00100000:2.0:1766098424.604705:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417180480:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098424.604741:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417180480:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 38us (203us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098424.604748:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1850
00000100:00100000:3.0:1766098424.605865:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098424.605925:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180544
00000100:00100000:2.0:1766098424.605966:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1115
00000100:00100000:2.0:1766098424.605972:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417180544:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098424.606086:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098424.606153:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417180544:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 180us (291us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098424.606160:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1115
00010000:00010000:1.0:1766098424.701850:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098425.277671:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098425.277677:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098425.277680:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098425.277687:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2928
00000100:00080000:2.0:1766098425.277690:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098425.277696:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098425.277713:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000047fe747c x1851885432864640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098425.277718:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432864576:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098425.277786:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098425.277818:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864640:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098425.277860:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098425.277903:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098425.277909:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864640
00000100:00100000:0.0:1766098425.277912:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432864704:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098425.277941:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098425.277946:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116
00000100:00100000:1.0:1766098425.277958:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432864640:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098425.278057:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098425.278108:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432864640:12345-0@lo:8:kworker/u8:0.0 Request processed in 149us (248us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098425.278115:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116
00000100:00100000:1.0:1766098425.278139:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432864512:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098425.278165:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098425.278182:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098425.278186:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432864768:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098425.278234:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098425.278264:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864512
00000100:00100000:1.0:1766098425.278294:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1263
00000100:00100000:1.0:1766098425.278298:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432864512:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098425.278339:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432864512:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (173us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098425.278345:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1263
00000100:00080000:1.0:1766098425.278382:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098425.278385:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098425.278389:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864640:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098425.278415:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432864512:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098425.278431:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864768
00000100:00100000:1.0:1766098425.278467:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 895
00000100:00100000:1.0:1766098425.278470:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432864768:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098425.278525:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432864768:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (290us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098425.278531:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 895
00010000:00080000:1.0:1766098425.278552:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008739e800 x1851885432864768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098432 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098425.278566:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008739e800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432864768:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098425.279342:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864576
00000100:00100000:2.0:1766098425.279378:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864704
00000100:00100000:2.0:1766098425.279400:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1851
00000100:00100000:2.0:1766098425.279405:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff866b23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432864576:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098425.279473:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff866b23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432864576:12345-0@lo:400:kworker/u8:0.0 Request processed in 66us (1688us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098425.279501:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1851
00000100:00100000:2.0:1766098425.279517:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1852
00000100:00100000:2.0:1766098425.279520:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000885bd0b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432864704:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098425.279550:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000885bd0b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432864704:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (1611us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098425.279555:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1852
00000100:00100000:0.0:1766098425.280294:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432864704:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098425.280382:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c21f3e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432864576:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766098425.727188:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098426.749647:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098427.773644:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098428.798821:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098429.734193:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098429.734245:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098429.734284:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098429.734323:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180672
00000100:00100000:1.0:1766098429.734386:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180608
00000100:00100000:2.0:1766098429.734391:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1853
00000100:00100000:2.0:1766098429.734398:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000538e714a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417180672:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098429.734433:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1264
00000100:00100000:1.0:1766098429.734439:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417180608:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098429.734524:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000538e714a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417180672:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 125us (279us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098429.734532:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1853
00000100:00100000:3.0:1766098429.736203:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180736
00000100:00100000:3.0:1766098429.736271:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1117
00000100:00100000:3.0:1766098429.736277:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d91044b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417180736:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098429.736375:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098429.736668:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d91044b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417180736:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 388us (2382us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098429.736677:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1117
00000100:00100000:1.0:1766098429.736753:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417180608:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2313us (2569us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098429.736762:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1264
00010000:00010000:0.0:1766098429.822879:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098430.398539:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098430.398546:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098430.398550:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:0.0:1766098430.398555:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766098430.398557:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2933
00000100:00080000:3.0:1766098430.398572:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:0.0:1766098430.398575:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f57e9d49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432864832:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098430.398580:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098430.398604:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e56ac6bf x1851885432864960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098430.398642:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098430.398667:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ab75beb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432865024:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098430.398678:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd66eec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432865088:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098430.398708:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098430.398716:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098430.398733:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098430.398741:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063bda73f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432864896:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098430.398762:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865024
00000100:00100000:3.0:1766098430.398766:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098430.398806:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864896
00000100:00100000:3.0:1766098430.398810:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865088
00000100:00100000:0.0:1766098430.398832:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1854
00000100:00100000:0.0:1766098430.398838:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d1139e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432865024:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098430.398846:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 896
00000100:00100000:3.0:1766098430.398851:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432865088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098430.398902:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432865088:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (187us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098430.398910:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 896
00000100:00100000:0.0:1766098430.398914:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d1139e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432865024:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (204us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098430.398922:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1854
00000100:00100000:0.0:1766098430.398942:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1855
00000100:00100000:0.0:1766098430.398946:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006522eee0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432864896:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098430.398970:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ab75beb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432865024:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098430.398982:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006522eee0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432864896:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (216us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098430.398990:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1855
00000100:00100000:3.0:1766098430.398998:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063bda73f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432864896:0@lo:400:kworker/u8:0.0
00010000:00080000:0.0:1766098430.399023:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bd66eec1 x1851885432865088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098437 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098430.399032:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd66eec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432865088:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098430.399100:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e56ac6bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864960:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098430.399132:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098430.399170:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864832
00000100:00100000:1.0:1766098430.399204:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1265
00000100:00100000:1.0:1766098430.399208:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432864832:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098430.399241:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432864832:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (601us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098430.399246:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1265
00000100:00100000:1.0:1766098430.399275:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f57e9d49 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432864832:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098430.402673:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432864960
00000100:00100000:3.0:1766098430.402732:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1118
00000100:00100000:3.0:1766098430.402738:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432864960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098430.402837:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098430.402918:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432864960:12345-0@lo:8:kworker/u8:0.0 Request processed in 179us (3785us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098430.402926:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1118
00000100:00080000:1.0:1766098430.403079:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098430.403084:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098430.403088:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e56ac6bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432864960:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766098430.846718:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098431.869714:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098432.893674:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098433.918498:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098434.845050:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098434.845103:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098434.845143:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098434.845200:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180928
00000100:00100000:2.0:1766098434.845227:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180864
00000100:00100000:3.0:1766098434.845253:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1119
00000100:00100000:3.0:1766098434.845261:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417180928:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:2.0:1766098434.845269:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1856
00000100:00100000:2.0:1766098434.845275:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000315ba24d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417180864:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098434.845336:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000315ba24d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417180864:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 61us (293us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098434.845343:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1856
00010000:02020000:3.0:1766098434.845356:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098434.845410:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417180928:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 150us (267us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098434.845417:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1119
00000100:00100000:0.0:1766098434.846276:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180800
00000100:00100000:0.0:1766098434.846315:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1266
00000100:00100000:0.0:1766098434.846320:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417180800:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098434.846355:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417180800:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (1252us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098434.846370:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1266
00010000:00010000:1.0:1766098434.942790:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098435.518295:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098435.518302:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098435.518304:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098435.518311:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2938
00000100:00080000:3.0:1766098435.518314:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098435.518319:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098435.518338:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004e8fcc9e x1851885432865280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098435.518390:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b1b751b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432865344:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098435.518451:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098435.518579:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa6a6aec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432865408:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098435.518609:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098435.518632:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098435.518637:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063bda73f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432865152:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098435.518662:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098435.518691:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865408
00000100:00100000:1.0:1766098435.518727:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 897
00000100:00100000:1.0:1766098435.518732:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432865408:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098435.518773:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432865408:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (163us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098435.518780:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 897
00010000:00080000:1.0:1766098435.518815:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fa6a6aec x1851885432865408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098442 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098435.518827:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa6a6aec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432865408:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098435.518861:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865152
00000100:00100000:1.0:1766098435.518881:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1267
00000100:00100000:1.0:1766098435.518884:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432865152:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098435.518910:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432865152:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (249us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098435.518916:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1267
00000100:00100000:0.0:1766098435.519932:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ab75beb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432865216:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098435.520020:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063bda73f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432865152:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098435.520044:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e8fcc9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432865280:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098435.520511:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098435.522550:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865280
00000100:00100000:2.0:1766098435.522604:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1120
00000100:00100000:2.0:1766098435.522611:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b413e52a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432865280:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098435.522744:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098435.522830:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b413e52a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432865280:12345-0@lo:8:kworker/u8:0.0 Request processed in 219us (2321us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098435.522837:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1120
00000100:00080000:2.0:1766098435.522880:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098435.522883:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098435.522887:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e8fcc9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432865280:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098435.525449:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865344
00000100:00100000:1.0:1766098435.525572:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1857
00000100:00100000:1.0:1766098435.525579:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c16b3689 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432865344:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098435.525662:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c16b3689 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432865344:12345-0@lo:400:kworker/u8:0.0 Request processed in 83us (7212us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098435.525700:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1857
00000100:00100000:3.0:1766098435.526052:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b1b751b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432865344:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098435.526466:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098435.526645:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865216
00000100:00100000:3.0:1766098435.526682:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1858
00000100:00100000:3.0:1766098435.526695:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043c8fc31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432865216:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098435.526749:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043c8fc31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432865216:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (284us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098435.526756:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1858
00000100:00100000:3.0:1766098435.526798:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ab75beb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432865216:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098435.966133:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098436.990873:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098438.013660:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098439.039975:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098439.965023:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098439.966877:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417180992
00000100:00100000:0.0:1766098439.967559:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1268
00000100:00100000:0.0:1766098439.967568:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417180992:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098439.967647:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417180992:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 80us (2631us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098439.967655:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1268
00000100:00100000:2.0:1766098439.970174:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098439.970282:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181056
00000100:00100000:2.0:1766098439.970320:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098439.970346:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1859
00000100:00100000:3.0:1766098439.970352:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008edde9a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417181056:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098439.970373:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181120
00000100:00100000:2.0:1766098439.970394:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1121
00000100:00100000:2.0:1766098439.970414:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c21f3e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417181120:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:3.0:1766098439.970433:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008edde9a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417181056:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 81us (267us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098439.970441:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1859
00010000:02020000:2.0:1766098439.970527:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098439.970557:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c21f3e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417181120:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 143us (238us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098439.970563:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1121
00010000:00010000:1.0:1766098440.061714:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098440.638220:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003523eb7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432865472:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098440.638294:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766098440.639727:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098440.639734:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098440.639737:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098440.639743:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2943
00000100:00080000:0.0:1766098440.639746:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098440.639752:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098440.639774:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000050b9c6b5 x1851885432865600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098440.639866:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098440.639876:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432865728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098440.639930:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098440.639995:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865472
00000100:00100000:3.0:1766098440.640045:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1269
00000100:00100000:3.0:1766098440.640053:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432865472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098440.640116:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432865472:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (1823us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098440.640124:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1269
00000100:00100000:3.0:1766098440.640162:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b028421 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432865536:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098440.640201:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098440.640230:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098440.640239:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432865664:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098440.640249:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865728
00000100:00100000:3.0:1766098440.640264:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098440.640302:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865536
00000100:00100000:1.0:1766098440.640334:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 898
00000100:00100000:1.0:1766098440.640343:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab75beb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432865728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098440.640353:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865664
00000100:00100000:3.0:1766098440.640403:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1860
00000100:00100000:3.0:1766098440.640408:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b1b751b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432865536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098440.640434:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab75beb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432865728:12345-0@lo:400:kworker/u8:0.0 Request processed in 91us (503us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098440.640444:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 898
00000100:00100000:0.0:1766098440.640673:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1861
00000100:00100000:0.0:1766098440.640682:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e189930e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432865664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098440.640750:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e189930e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432865664:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (478us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098440.640767:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1861
00000100:00100000:1.0:1766098440.640831:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432865664:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098440.640843:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432865600:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098440.640883:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098440.640937:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865600
00000100:00100000:1.0:1766098440.640979:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1122
00000100:00100000:1.0:1766098440.640985:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000433d0f0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432865600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098440.641085:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098440.641132:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000433d0f0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432865600:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (249us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098440.641139:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1122
00010000:00080000:2.0:1766098440.641735:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003a108df5 x1851885432865728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098447 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098440.641749:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432865728:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098440.641762:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003523eb7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432865472:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098440.642065:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098440.642071:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098440.642075:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432865600:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098440.642341:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b1b751b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432865536:12345-0@lo:400:kworker/u8:0.0 Request processed in 1928us (2136us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098440.642350:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1860
00000100:00100000:1.0:1766098440.642945:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b028421 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432865536:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098441.089450:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098442.117108:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098443.134262:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098444.158821:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098445.090365:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098445.090711:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098445.090760:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098445.090799:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181248
00000100:00100000:2.0:1766098445.090811:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181312
00000100:00100000:1.0:1766098445.090866:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1862
00000100:00100000:2.0:1766098445.090872:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1123
00000100:00100000:1.0:1766098445.090877:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008ebdb4c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417181248:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098445.090880:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417181312:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:0.0:1766098445.090944:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181184
00000100:00100000:0.0:1766098445.090976:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1270
00000100:00100000:1.0:1766098445.090980:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008ebdb4c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417181248:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 103us (269us total) trans 0 rc 0/0
00010000:02020000:2.0:1766098445.090991:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098445.091007:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1862
00000100:00100000:2.0:1766098445.091030:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417181312:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 151us (270us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098445.091039:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1123
00000100:00100000:0.0:1766098445.091091:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417181184:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098445.091120:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417181184:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 138us (769us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098445.091126:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1270
00010000:00010000:0.0:1766098445.182379:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098445.763009:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432865792:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098445.763098:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098445.763446:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865792
00000100:00100000:0.0:1766098445.765003:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1271
00000100:00100000:0.0:1766098445.765016:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432865792:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098445.765329:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098445.765337:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098445.765341:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098445.765347:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2949
00000100:00080000:3.0:1766098445.765350:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098445.765355:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098445.765377:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b2232645 x1851885432865920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098445.765432:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006394e12b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432866048:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765474:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098445.765530:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060c730a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432865856:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765556:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098445.765571:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098445.765576:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d48a9336 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432865984:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765594:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098445.765617:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432865920:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765637:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098445.765698:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865856
00000100:00100000:1.0:1766098445.765715:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865984
00000100:00100000:3.0:1766098445.765741:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1863
00000100:00100000:3.0:1766098445.765746:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000633e633e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432865856:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098445.765755:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1864
00000100:00100000:1.0:1766098445.765760:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432865984:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765792:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000633e633e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432865856:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (235us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098445.765800:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1863
00000100:00100000:1.0:1766098445.765805:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c2623e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432865984:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (209us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098445.765812:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1864
00000100:00100000:3.0:1766098445.765831:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866048
00000100:00100000:1.0:1766098445.765844:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432865920
00000100:00100000:3.0:1766098445.765861:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 899
00000100:00100000:3.0:1766098445.765866:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6c1f26a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432866048:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098445.765871:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124
00000100:00100000:1.0:1766098445.765876:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432865920:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765901:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6c1f26a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432866048:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (427us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098445.765907:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 899
00000100:00100000:3.0:1766098445.765947:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d48a9336 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432865984:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098445.765956:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060c730a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432865856:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766098445.765982:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:3.0:1766098445.765991:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006394e12b x1851885432866048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098452 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098445.766000:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006394e12b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432866048:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098445.766021:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432865920:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (383us total) trans 0 rc -19/-19
00000100:00080000:3.0:1766098445.766026:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:1.0:1766098445.766030:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124
00000100:00080000:3.0:1766098445.766033:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098445.766038:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432865920:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098445.766495:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432865792:12345-0@lo:400:kworker/u8:2.0 Request processed in 1481us (3397us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098445.766505:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1271
00000100:00100000:2.0:1766098445.766522:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432865792:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098446.209880:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098447.232596:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098448.256063:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098449.278125:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098450.214501:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098450.214556:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098450.214618:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098450.214758:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181376
00000100:00100000:0.0:1766098450.214840:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1272
00000100:00100000:0.0:1766098450.214848:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417181376:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098450.214960:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417181376:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 112us (342us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098450.214970:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1272
00000100:00100000:2.0:1766098450.215113:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181504
00000100:00100000:2.0:1766098450.215153:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1125
00000100:00100000:2.0:1766098450.215158:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417181504:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098450.215277:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098450.215315:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417181504:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 156us (760us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098450.215323:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1125
00000100:00100000:3.0:1766098450.218230:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181440
00000100:00100000:3.0:1766098450.218336:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1865
00000100:00100000:3.0:1766098450.218345:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a9ccd09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417181440:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098450.218440:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a9ccd09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417181440:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 95us (3969us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098450.218466:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1865
00010000:00010000:1.0:1766098450.301739:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098450.877798:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098450.877805:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098450.877808:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098450.877815:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2954
00000100:00080000:0.0:1766098450.877818:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098450.877824:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098450.877842:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b0ded71f x1851885432866240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098450.877919:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432866112:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.877980:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098450.878017:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098450.878024:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432866368:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098450.878033:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432866176:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878050:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098450.878078:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cfd14c2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432866304:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878100:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098450.878133:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866304
00000100:00100000:3.0:1766098450.878176:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1866
00000100:00100000:3.0:1766098450.878182:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006394e12b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432866304:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878240:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006394e12b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432866304:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (137us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098450.878257:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1866
00000100:00100000:3.0:1766098450.878288:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866368
00000100:00100000:3.0:1766098450.878328:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900
00000100:00100000:3.0:1766098450.878333:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2232645 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432866368:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878378:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2232645 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432866368:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (331us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098450.878386:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900
00000100:00100000:3.0:1766098450.878410:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432866240:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878435:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098450.878510:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cfd14c2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432866304:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878545:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866240
00000100:00100000:3.0:1766098450.878588:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1126
00000100:00100000:3.0:1766098450.878592:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060c730a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432866240:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098450.878680:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:2.0:1766098450.878695:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f42693bf x1851885432866368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098457 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098450.878713:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432866368:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878732:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060c730a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432866240:12345-0@lo:8:kworker/u8:0.0 Request processed in 137us (295us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098450.878742:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1126
00000100:00100000:1.0:1766098450.878761:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:3.0:1766098450.878789:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098450.878795:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098450.878803:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432866240:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878837:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866176
00000100:00100000:3.0:1766098450.878877:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1867
00000100:00100000:3.0:1766098450.878881:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac943495 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432866176:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098450.878929:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac943495 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432866176:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (171us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098450.878935:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1867
00000100:00100000:0.0:1766098450.879010:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866112
00000100:00100000:0.0:1766098450.879040:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1273
00000100:00100000:0.0:1766098450.879044:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432866112:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098450.879077:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432866112:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (1099us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098450.879083:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1273
00000100:00100000:0.0:1766098450.879296:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432866112:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098450.879429:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432866176:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098451.325730:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098452.349840:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098453.373677:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098454.397811:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098455.328368:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098455.328425:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098455.328557:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181696
00000100:00100000:2.0:1766098455.328590:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098455.328641:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1127
00000100:00100000:3.0:1766098455.328649:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417181696:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:1.0:1766098455.328715:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181632
00010000:02020000:3.0:1766098455.328755:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098455.328779:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1868
00000100:00100000:1.0:1766098455.328789:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008739e800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417181632:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098455.328860:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417181696:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 209us (497us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098455.328878:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1127
00000100:00100000:1.0:1766098455.328880:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008739e800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417181632:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 83us (283us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098455.328892:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1868
00000100:00100000:0.0:1766098455.331401:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181568
00000100:00100000:0.0:1766098455.331535:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1274
00000100:00100000:0.0:1766098455.331543:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417181568:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098455.331634:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417181568:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 91us (3210us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098455.331656:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1274
00010000:00010000:1.0:1766098455.423858:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098455.997753:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098455.998392:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432866432:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098455.998511:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098455.998616:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866432
00000100:00100000:3.0:1766098455.998678:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1275
00000100:00100000:3.0:1766098455.998685:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432866432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098455.998771:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432866432:12345-0@lo:400:kworker/u8:0.0 Request processed in 85us (262us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098455.998779:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1275
00000100:00100000:3.0:1766098455.998842:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432866432:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098455.999363:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098455.999370:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098455.999374:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098455.999380:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2959
00000100:00080000:0.0:1766098455.999383:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098455.999387:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098455.999412:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000044763d3d x1851885432866560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098455.999465:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043119846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432866496:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999529:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098455.999555:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098455.999560:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002af2c706 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432866624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999581:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098455.999613:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866496
00000100:00100000:0.0:1766098455.999650:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866624
00000100:00100000:0.0:1766098455.999670:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1869
00000100:00100000:0.0:1766098455.999674:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b24273d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432866496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999720:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b24273d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432866496:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (191us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098455.999727:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1869
00000100:00100000:0.0:1766098455.999745:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1870
00000100:00100000:0.0:1766098455.999748:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b510690 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432866624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999773:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b510690 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432866624:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (192us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098455.999792:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1870
00000100:00100000:0.0:1766098455.999837:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002af2c706 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432866624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999847:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043119846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432866496:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999868:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432866688:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999897:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098455.999957:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000044763d3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432866560:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098455.999981:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098456.000016:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866688
00000100:00100000:0.0:1766098456.000039:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866560
00000100:00100000:0.0:1766098456.000056:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 901
00000100:00100000:0.0:1766098456.000060:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043119846 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432866688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098456.000091:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043119846 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432866688:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (195us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098456.000098:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 901
00000100:00100000:0.0:1766098456.000108:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128
00000100:00100000:0.0:1766098456.000111:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432866560:12345-0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766098456.000985:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004dffa5d1 x1851885432866688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098462 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098456.001002:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432866688:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766098456.001966:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098456.002068:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432866560:12345-0@lo:8:kworker/u8:0.0 Request processed in 1954us (2085us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098456.002083:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128
00000100:00080000:1.0:1766098456.002108:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098456.002114:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098456.002120:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000044763d3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432866560:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766098456.447677:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098457.469816:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098458.497134:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098459.526146:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098460.450395:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098460.450524:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181888
00000100:00100000:0.0:1766098460.450590:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1129
00000100:00100000:0.0:1766098460.450599:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047fe747c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417181888:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098460.450729:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098460.450832:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047fe747c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417181888:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 232us (449us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098460.450842:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1129
00000100:00100000:3.0:1766098460.450898:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098460.451094:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098460.451165:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181824
00000100:00100000:3.0:1766098460.451226:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1871
00000100:00100000:3.0:1766098460.451282:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417181824:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098460.451342:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000368c1422 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417181824:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 60us (248us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098460.451348:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1871
00000100:00100000:0.0:1766098460.458734:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181760
00000100:00100000:0.0:1766098460.458832:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1276
00000100:00100000:0.0:1766098460.458840:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417181760:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098460.458958:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417181760:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 118us (8067us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098460.458965:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1276
00010000:00010000:0.0:1766098460.544825:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098461.117726:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098461.117735:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098461.117739:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098461.117749:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2964
00000100:00080000:2.0:1766098461.117754:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:3.0:1766098461.117756:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000059cbb111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432866752:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766098461.117762:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098461.117788:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f827ef16 x1851885432866880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098461.117848:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098461.117887:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000059a3ed9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432867008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098461.117924:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f827ef16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432866880:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098461.117931:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098461.117961:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098461.118012:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ccf06dd7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432866816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098461.118045:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098461.118065:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098461.118070:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432866944:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098461.118087:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098461.118146:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866752
00000100:00100000:2.0:1766098461.118208:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1277
00000100:00100000:2.0:1766098461.118215:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432866752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098461.118277:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432866752:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (429us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098461.118306:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1277
00000100:00100000:2.0:1766098461.118357:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866816
00000100:00100000:2.0:1766098461.118385:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866944
00000100:00100000:2.0:1766098461.118407:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1872
00000100:00100000:2.0:1766098461.118410:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a75d9f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432866816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098461.118448:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a75d9f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432866816:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (402us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098461.118455:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1872
00000100:00100000:2.0:1766098461.118468:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1873
00000100:00100000:2.0:1766098461.118473:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000914f119b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432866944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098461.118523:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000914f119b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432866944:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (435us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098461.118529:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1873
00000100:00100000:2.0:1766098461.118570:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432866944:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098461.118581:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ccf06dd7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432866816:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098461.118618:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000059cbb111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432866752:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098461.118994:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867008
00000100:00100000:1.0:1766098461.119039:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432866880
00000100:00100000:1.0:1766098461.119065:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 902
00000100:00100000:1.0:1766098461.119080:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432867008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098461.119125:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432867008:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (1195us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098461.119133:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 902
00000100:00100000:3.0:1766098461.119148:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1130
00000100:00100000:3.0:1766098461.119154:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d48a9336 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432866880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098461.119276:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:0.0:1766098461.119296:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000059a3ed9e x1851885432867008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098468 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098461.119309:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000059a3ed9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432867008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098461.119314:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d48a9336 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432866880:12345-0@lo:8:kworker/u8:2.0 Request processed in 158us (1354us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098461.119651:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1130
00000100:00080000:3.0:1766098461.119684:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098461.119688:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098461.119698:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f827ef16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432866880:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766098461.565771:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098462.591796:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098463.613726:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098464.637918:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098465.569788:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098465.569846:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098465.569907:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098465.569929:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417181952
00000100:00100000:2.0:1766098465.569929:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182016
00000100:00100000:2.0:1766098465.570002:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1874
00000100:00100000:0.0:1766098465.570003:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1278
00000100:00100000:0.0:1766098465.570015:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417181952:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098465.570016:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417182016:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098465.570057:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182080
00000100:00100000:0.0:1766098465.570094:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417181952:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 82us (317us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098465.570095:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417182016:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 84us (249us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098465.570105:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1874
00000100:00100000:0.0:1766098465.570116:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1278
00000100:00100000:3.0:1766098465.570128:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1131
00000100:00100000:3.0:1766098465.570148:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417182080:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098465.570256:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098465.570314:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417182080:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 167us (413us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098465.570322:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1131
00010000:00010000:1.0:1766098465.663364:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098466.239612:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098466.239618:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098466.239622:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098466.239629:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2969
00000100:00080000:0.0:1766098466.239632:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098466.239638:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098466.239664:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a48ec222 x1851885432867200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098466.239707:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432867264:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098466.239765:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098466.239795:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098466.239800:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432867136:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098466.239821:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098466.239904:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432867328:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098466.239916:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b77e32c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432867072:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098466.239941:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098466.239997:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867264
00000100:00100000:3.0:1766098466.240001:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098466.240049:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867136
00000100:00100000:3.0:1766098466.240068:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a48ec222 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432867200:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098466.240077:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1875
00000100:00100000:1.0:1766098466.240084:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000044763d3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432867264:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098466.240104:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098466.240172:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867328
00000100:00100000:1.0:1766098466.240178:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000044763d3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432867264:12345-0@lo:400:kworker/u8:2.0 Request processed in 91us (410us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098466.240186:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1875
00000100:00100000:1.0:1766098466.240210:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1876
00000100:00100000:1.0:1766098466.240215:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dffa5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432867136:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098466.240224:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867200
00000100:00100000:3.0:1766098466.240255:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 903
00000100:00100000:3.0:1766098466.240262:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432867328:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098466.240269:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dffa5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432867136:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (440us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098466.240279:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1876
00000100:00100000:3.0:1766098466.240327:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432867328:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (386us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098466.240336:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 903
00000100:00100000:1.0:1766098466.240350:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432867136:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098466.240359:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1132
00000100:00100000:3.0:1766098466.240364:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432867200:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098466.240395:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432867264:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098466.240451:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867072
00010000:02020000:3.0:1766098466.240471:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098466.240511:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1279
00000100:00100000:1.0:1766098466.240517:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432867072:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098466.240562:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432867200:12345-0@lo:8:kworker/u8:2.0 Request processed in 195us (457us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098466.240580:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432867072:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (581us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098466.240600:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1132
00000100:00100000:1.0:1766098466.240602:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1279
00000100:00100000:1.0:1766098466.240677:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b77e32c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432867072:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098466.240712:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098466.240716:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00010000:00080000:3.0:1766098466.240720:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000d48768b x1851885432867328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098473 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766098466.240721:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a48ec222 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432867200:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098466.240741:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432867328:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098466.685952:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098467.709695:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098468.734833:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098469.759295:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098470.685185:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098470.685276:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182272
00000100:00100000:0.0:1766098470.685330:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1133
00000100:00100000:0.0:1766098470.685337:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b57ab98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417182272:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098470.685431:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098470.689185:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098470.689335:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098470.689374:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182144
00000100:00100000:2.0:1766098470.689420:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1280
00000100:00100000:2.0:1766098470.689427:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417182144:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098470.689510:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417182144:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 82us (175us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098470.689518:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1280
00000100:00100000:3.0:1766098470.689710:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182208
00000100:00100000:3.0:1766098470.689783:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1877
00000100:00100000:3.0:1766098470.689787:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059cbb111 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417182208:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098470.689821:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059cbb111 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417182208:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (645us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098470.689828:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1877
00000100:00100000:0.0:1766098470.690113:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b57ab98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417182272:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 4774us (4934us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098470.690123:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1133
00010000:00010000:2.0:1766098470.781758:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098471.357701:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098471.357723:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098471.357726:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098471.357734:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2974
00000100:00080000:0.0:1766098471.357738:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098471.357743:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098471.357767:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b0ded71f x1851885432867520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098471.357823:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cfd14c2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432867648:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098471.357823:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432867584:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358205:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098471.358239:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098471.358244:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432867392:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358271:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098471.358308:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867648
00000100:00100000:0.0:1766098471.358350:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 904
00000100:00100000:0.0:1766098471.358356:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b028421 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432867648:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358408:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b028421 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432867648:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (204us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098471.358418:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 904
00010000:00080000:0.0:1766098471.358464:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cfd14c2b x1851885432867648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098478 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098471.358474:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432867456:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358504:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cfd14c2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432867648:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358550:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432867520:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098471.358567:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098471.358602:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098471.358644:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867392
00000100:00100000:0.0:1766098471.358652:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867520
00000100:00100000:0.0:1766098471.358684:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1134
00000100:00100000:0.0:1766098471.358697:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432867520:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098471.358789:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098471.358823:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432867520:12345-0@lo:8:kworker/u8:2.0 Request processed in 126us (223us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098471.358829:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1134
00000100:00080000:0.0:1766098471.358851:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098471.358854:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098471.358857:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432867520:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358892:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867456
00000100:00100000:0.0:1766098471.358919:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1878
00000100:00100000:0.0:1766098471.358923:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ccf06dd7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432867456:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098471.358956:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ccf06dd7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432867456:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (392us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098471.358962:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1878
00000100:00100000:2.0:1766098471.359430:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1281
00000100:00100000:2.0:1766098471.359440:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432867392:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098471.359543:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432867392:12345-0@lo:400:kworker/u8:2.0 Request processed in 105us (1271us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098471.359553:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1281
00000100:00100000:1.0:1766098471.359849:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098471.359899:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867584
00000100:00100000:2.0:1766098471.359962:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432867392:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098471.360025:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1879
00000100:00100000:1.0:1766098471.360033:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020a14350 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432867584:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098471.360102:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020a14350 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432867584:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (246us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098471.360109:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1879
00000100:00100000:1.0:1766098471.360166:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432867584:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098471.360212:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432867456:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098471.805757:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098472.830028:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098473.853621:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098474.880053:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098475.805729:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098475.805839:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182336
00000100:00100000:3.0:1766098475.805894:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1282
00000100:00100000:3.0:1766098475.805900:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417182336:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098475.805979:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417182336:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (259us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098475.805986:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1282
00000100:00100000:1.0:1766098475.806960:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098475.807052:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182400
00000100:00100000:0.0:1766098475.807111:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1880
00000100:00100000:0.0:1766098475.807116:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6e93a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417182400:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098475.807174:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6e93a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417182400:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 57us (217us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098475.807180:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1880
00000100:00100000:0.0:1766098475.810401:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098475.810513:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182464
00000100:00100000:3.0:1766098475.810574:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1135
00000100:00100000:3.0:1766098475.810583:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417182464:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098475.810705:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098475.810779:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417182464:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 197us (388us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098475.810788:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1135
00010000:00010000:0.0:1766098475.903644:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098476.477819:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098476.477826:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098476.477829:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098476.477837:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2979
00000100:00080000:1.0:1766098476.477840:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098476.477845:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098476.477866:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005e10322b x1851885432867840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098476.477914:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432867904:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.477975:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098476.478002:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098476.478009:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432867776:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478031:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098476.478079:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432867840:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478107:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098476.478142:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867904
00000100:00100000:1.0:1766098476.478179:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867776
00000100:00100000:1.0:1766098476.478199:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1881
00000100:00100000:1.0:1766098476.478204:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096e289f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432867904:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478249:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096e289f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432867904:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (274us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098476.478256:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1881
00000100:00100000:1.0:1766098476.478277:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1882
00000100:00100000:0.0:1766098476.478295:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432867712:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478295:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432867776:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478321:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432867776:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (291us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098476.478327:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1882
00000100:00100000:0.0:1766098476.478331:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098476.478346:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867840
00000100:00100000:0.0:1766098476.478348:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098476.478352:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040addeb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432867968:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478362:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1136
00000100:00100000:1.0:1766098476.478365:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c51db5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432867840:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098476.478372:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098476.478393:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867968
00000100:00100000:0.0:1766098476.478415:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 905
00000100:00100000:0.0:1766098476.478418:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432867968:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098476.478445:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432867968:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (72us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098476.478450:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 905
00010000:02020000:1.0:1766098476.478459:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:0.0:1766098476.478499:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000040addeb2 x1851885432867968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098483 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098476.478511:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040addeb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432867968:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098476.478513:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c51db5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432867840:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (406us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098476.478519:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1136
00000100:00080000:1.0:1766098476.478546:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098476.478549:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098476.478552:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432867840:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098476.478563:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432867776:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098476.478573:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432867904:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098476.479744:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432867712
00000100:00100000:2.0:1766098476.479799:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1283
00000100:00100000:2.0:1766098476.479804:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432867712:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098476.479874:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432867712:12345-0@lo:400:kworker/u8:0.0 Request processed in 69us (1543us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098476.479891:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1283
00000100:00100000:1.0:1766098476.479899:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432867712:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098476.928415:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098477.951248:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098478.975762:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098479.998612:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098480.924746:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098480.926801:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098480.928404:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182656
00000100:00100000:0.0:1766098480.928575:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182592
00000100:00100000:2.0:1766098480.928596:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098480.928620:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1883
00000100:00100000:0.0:1766098480.928628:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c21f3e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417182592:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098480.928632:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182528
00000100:00100000:2.0:1766098480.928663:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1284
00000100:00100000:2.0:1766098480.928668:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417182528:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098480.928704:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c21f3e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417182592:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 73us (1907us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098480.928713:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1883
00000100:00100000:2.0:1766098480.928718:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417182528:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 33us (110us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098480.928726:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1284
00000100:00100000:3.0:1766098480.929120:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1137
00000100:00100000:3.0:1766098480.929125:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417182656:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098480.929215:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098480.930064:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417182656:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 936us (5323us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098480.930073:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1137
00010000:00010000:2.0:1766098481.024495:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098481.597980:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432868032:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098481.598000:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098481.598006:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098481.598010:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098481.598026:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2984
00000100:00080000:3.0:1766098481.598029:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098481.598036:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098481.598060:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004e8fcc9e x1851885432868160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098481.598067:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098481.598112:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098481.598115:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ab75beb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432868096:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098481.598122:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa6a6aec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432868288:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098481.598134:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868032
00000100:00100000:3.0:1766098481.598157:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098481.598164:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098481.598185:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098481.598194:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063bda73f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432868224:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098481.598229:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1285
00000100:00100000:3.0:1766098481.598234:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098481.598237:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432868032:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098481.598257:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868096
00000100:00100000:3.0:1766098481.598274:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868224
00000100:00100000:0.0:1766098481.598294:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884
00000100:00100000:2.0:1766098481.598299:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432868032:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (235us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098481.598303:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e56ac6bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432868096:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098481.598309:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1885
00000100:00100000:2.0:1766098481.598314:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1285
00000100:00100000:3.0:1766098481.598318:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f57e9d49 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432868224:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098481.598353:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e56ac6bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432868096:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (196us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098481.598361:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884
00000100:00100000:3.0:1766098481.598365:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f57e9d49 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432868224:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (131us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098481.598382:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1885
00000100:00100000:0.0:1766098481.598404:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432868032:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098481.598427:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063bda73f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432868224:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098481.598448:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ab75beb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432868096:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098481.598503:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868288
00000100:00100000:3.0:1766098481.598536:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 906
00000100:00100000:3.0:1766098481.598540:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432868288:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098481.598595:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432868288:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (432us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098481.598606:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 906
00000100:00100000:1.0:1766098481.598760:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e8fcc9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432868160:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098481.598790:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098481.598901:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868160
00000100:00100000:0.0:1766098481.598934:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1138
00000100:00100000:0.0:1766098481.598938:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432868160:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098481.599046:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:2.0:1766098481.599122:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fa6a6aec x1851885432868288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098488 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098481.599133:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa6a6aec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432868288:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098481.599154:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432868160:12345-0@lo:8:kworker/u8:2.0 Request processed in 212us (362us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098481.599161:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1138
00000100:00080000:1.0:1766098481.599343:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098481.599347:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098481.599351:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e8fcc9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432868160:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766098482.045706:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098483.069744:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098484.093719:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098485.117796:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098486.047659:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098486.047711:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098486.047749:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098486.047850:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182784
00000100:00100000:1.0:1766098486.047912:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1886
00000100:00100000:1.0:1766098486.047918:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f827ef16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417182784:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098486.048018:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f827ef16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417182784:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 98us (366us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098486.048026:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1886
00000100:00100000:0.0:1766098486.048105:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182720
00000100:00100000:0.0:1766098486.048141:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1286
00000100:00100000:0.0:1766098486.048145:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417182720:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098486.048171:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417182720:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 26us (462us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098486.048177:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1286
00000100:00100000:3.0:1766098486.048387:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182848
00000100:00100000:3.0:1766098486.048420:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1139
00000100:00100000:3.0:1766098486.048424:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417182848:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098486.050746:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098486.050883:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417182848:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 2455us (3131us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098486.050901:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1139
00010000:00010000:2.0:1766098486.141706:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098486.717775:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098486.717782:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098486.717786:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098486.717794:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2990
00000100:00080000:1.0:1766098486.717798:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098486.717803:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098486.717826:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e445b90d x1851885432868480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098486.717830:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063ac9a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432868416:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098486.717881:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432868544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098486.717903:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098486.717923:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432868480:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098486.717954:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098486.717998:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098486.718020:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868480
00000100:00100000:1.0:1766098486.718032:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868416
00000100:00100000:3.0:1766098486.718057:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1140
00000100:00100000:3.0:1766098486.718064:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432868480:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098486.718070:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868544
00000100:00100000:1.0:1766098486.718091:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1887
00000100:00100000:1.0:1766098486.718095:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab75beb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432868416:12345-0@lo:400:kworker/u8:0.0
00010000:02020000:3.0:1766098486.718152:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098486.718166:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab75beb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432868416:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (248us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098486.718175:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1887
00000100:00100000:3.0:1766098486.718191:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432868480:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (236us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098486.718198:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1888
00000100:00100000:1.0:1766098486.718201:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d11b33f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432868544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098486.718215:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1140
00000100:00100000:1.0:1766098486.718250:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d11b33f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432868544:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (251us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098486.718257:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063ac9a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432868416:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098486.718259:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1888
00000100:00100000:0.0:1766098486.718331:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df8193b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432868352:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098486.718359:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098486.718435:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432868608:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098486.718522:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432868544:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098486.718547:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098486.718550:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098486.718553:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432868480:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098486.719353:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098486.719463:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868352
00000100:00100000:1.0:1766098486.719524:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1287
00000100:00100000:1.0:1766098486.719529:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432868352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098486.719579:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432868352:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (1219us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098486.719587:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1287
00000100:00100000:1.0:1766098486.719637:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df8193b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432868352:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098486.720344:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868608
00000100:00100000:3.0:1766098486.721807:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 907
00000100:00100000:3.0:1766098486.721816:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc68c7d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432868608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098486.721888:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc68c7d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432868608:12345-0@lo:400:kworker/u8:0.0 Request processed in 73us (2538us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098486.721894:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 907
00010000:00080000:3.0:1766098486.721940:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d5337db0 x1851885432868608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098493 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098486.721952:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432868608:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098487.168702:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098488.190659:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098489.216535:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098490.238580:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098491.168821:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098491.168860:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098491.168893:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098491.168979:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182976
00000100:00100000:3.0:1766098491.169031:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1889
00000100:00100000:3.0:1766098491.169037:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff2504ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417182976:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098491.169103:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff2504ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417182976:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 66us (290us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098491.169110:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1889
00000100:00100000:1.0:1766098491.169173:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417182912
00000100:00100000:1.0:1766098491.169201:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1288
00000100:00100000:1.0:1766098491.169205:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417182912:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098491.169230:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417182912:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 24us (370us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098491.169235:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1288
00000100:00100000:0.0:1766098491.169335:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183040
00000100:00100000:0.0:1766098491.169368:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1141
00000100:00100000:0.0:1766098491.169372:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417183040:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098491.169467:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098491.169526:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417183040:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 153us (633us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098491.169532:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1141
00010000:00010000:0.0:1766098491.261694:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098491.839258:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098491.839265:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098491.839269:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098491.839275:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 2995
00000100:00080000:0.0:1766098491.839279:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098491.839285:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098491.839303:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f42693bf x1851885432868800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098491.839390:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432868864:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098491.839446:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098491.839488:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098491.839493:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432868736:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098491.839515:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098491.839630:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432868672:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098491.839661:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098491.839983:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432868928:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098491.840012:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098491.840060:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868864
00000100:00100000:3.0:1766098491.840094:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868736
00000100:00100000:3.0:1766098491.840109:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1890
00000100:00100000:3.0:1766098491.840114:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a48ec222 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432868864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098491.840157:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a48ec222 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432868864:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (711us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098491.840171:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1890
00000100:00100000:3.0:1766098491.840186:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1891
00000100:00100000:3.0:1766098491.840189:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b77e32c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432868736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098491.840211:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b77e32c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432868736:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (696us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098491.840216:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1891
00000100:00100000:3.0:1766098491.840250:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432868736:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098491.840260:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432868864:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098491.840281:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868928
00000100:00100000:3.0:1766098491.840303:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908
00000100:00100000:3.0:1766098491.840307:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063bda73f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432868928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098491.840336:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063bda73f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432868928:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (324us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098491.840343:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908
00010000:00080000:3.0:1766098491.840364:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000d48768b x1851885432868928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098498 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098491.840373:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432868928:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098491.840699:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868672
00000100:00100000:0.0:1766098491.840732:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1289
00000100:00100000:0.0:1766098491.840735:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432868672:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098491.840773:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432868672:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (1112us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098491.840779:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1289
00000100:00100000:1.0:1766098491.841457:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432868800:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098491.841563:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098491.841636:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432868672:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098491.843576:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868800
00000100:00100000:0.0:1766098491.843634:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1142
00000100:00100000:0.0:1766098491.843640:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df8193b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432868800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098491.843734:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098491.843800:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df8193b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432868800:12345-0@lo:8:kworker/u8:0.0 Request processed in 160us (2238us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098491.843808:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1142
00000100:00080000:1.0:1766098491.844211:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098491.844228:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098491.844233:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432868800:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098492.287020:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098493.309881:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098494.334947:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098495.357650:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098496.290081:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098496.290130:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098496.290225:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098496.290326:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183104
00000100:00100000:2.0:1766098496.290378:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1290
00000100:00100000:2.0:1766098496.290385:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417183104:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098496.290452:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417183104:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 66us (379us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098496.290459:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1290
00000100:00100000:1.0:1766098496.290667:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183168
00000100:00100000:1.0:1766098496.290706:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1892
00000100:00100000:1.0:1766098496.290710:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000678c16b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417183168:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098496.290757:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000678c16b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417183168:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 46us (627us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098496.290764:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1892
00000100:00100000:1.0:1766098496.290785:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183232
00000100:00100000:1.0:1766098496.290805:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1143
00000100:00100000:1.0:1766098496.290809:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183232:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098496.290903:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098496.290939:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183232:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 129us (714us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098496.290945:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1143
00010000:00010000:2.0:1766098496.382445:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098496.957694:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098496.957702:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098496.957705:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098496.957712:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3000
00000100:00080000:1.0:1766098496.957715:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098496.957721:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098496.957742:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006df27fbb x1851885432869120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098496.957792:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432869184:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.957866:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098496.957897:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098496.957915:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432869056:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.957939:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098496.957961:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006df27fbb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432869120:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098496.957997:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098496.958029:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869184
00000100:00100000:1.0:1766098496.958064:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869056
00000100:00100000:1.0:1766098496.958080:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1893
00000100:00100000:1.0:1766098496.958085:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db0fcd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432869184:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.958131:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db0fcd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432869184:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (265us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098496.958138:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1893
00000100:00100000:1.0:1766098496.958176:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1894
00000100:00100000:1.0:1766098496.958181:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432869056:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.958206:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432869056:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (267us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098496.958211:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1894
00000100:00100000:1.0:1766098496.958243:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432869056:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.958253:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432869184:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.958275:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869120
00000100:00100000:1.0:1766098496.958294:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1144
00000100:00100000:1.0:1766098496.958298:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432869120:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098496.958405:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098496.958441:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432869120:12345-0@lo:8:kworker/u8:0.0 Request processed in 142us (444us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098496.958448:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1144
00000100:00080000:1.0:1766098496.958473:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098496.958501:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098496.958505:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006df27fbb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432869120:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098496.958597:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432869248:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098496.958627:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098496.958651:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098496.958657:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432868992:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098496.958680:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098496.958704:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432868992
00000100:00100000:0.0:1766098496.958726:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1291
00000100:00100000:0.0:1766098496.958730:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432868992:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098496.958759:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432868992:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (79us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098496.958772:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1291
00000100:00100000:0.0:1766098496.958804:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432868992:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.959721:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869248
00000100:00100000:1.0:1766098496.959765:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 909
00000100:00100000:1.0:1766098496.959770:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f42693bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432869248:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098496.959833:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f42693bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432869248:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (1205us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098496.959840:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 909
00010000:00080000:2.0:1766098496.959922:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002ec3fedd x1851885432869248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098503 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098496.959934:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432869248:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098497.405704:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098498.429703:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098499.453820:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098500.478451:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098501.404812:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098501.404862:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098501.404982:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098501.405000:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183296
00000100:00100000:2.0:1766098501.405084:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1292
00000100:00100000:2.0:1766098501.405091:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417183296:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098501.405093:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183424
00000100:00100000:1.0:1766098501.405142:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1145
00000100:00100000:1.0:1766098501.405150:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183424:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:2.0:1766098501.405183:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417183296:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (319us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098501.405192:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1292
00010000:02020000:1.0:1766098501.405251:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098501.405637:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183360
00000100:00100000:3.0:1766098501.405687:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1895
00000100:00100000:3.0:1766098501.405692:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad5bbc91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417183360:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098501.405832:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad5bbc91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417183360:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 138us (1031us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098501.405857:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1895
00000100:00100000:1.0:1766098501.419574:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183424:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 14411us (14583us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098501.419597:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1145
00010000:00010000:2.0:1766098501.502141:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098502.079759:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098502.079766:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098502.079770:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098502.079780:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3005
00000100:00080000:3.0:1766098502.079784:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098502.079789:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098502.079812:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000fa6a6aec x1851885432869440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098502.079875:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e8fcc9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432869376:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.079900:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432869568:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098502.079953:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098502.080031:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd66eec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432869312:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.081857:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098502.082015:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869376
00000100:00100000:2.0:1766098502.082075:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896
00000100:00100000:2.0:1766098502.082083:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432869376:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.085903:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab4c82a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432869376:12345-0@lo:400:kworker/u8:2.0 Request processed in 3814us (5947us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098502.085918:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896
00000100:00100000:2.0:1766098502.086173:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa6a6aec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432869440:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098502.086225:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098502.086271:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432869504:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.086300:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098502.086342:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869568
00000100:00100000:2.0:1766098502.086389:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869440
00000100:00100000:2.0:1766098502.086410:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 910
00000100:00100000:2.0:1766098502.086415:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432869568:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.086499:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432869568:12345-0@lo:400:kworker/u8:2.0 Request processed in 82us (4649us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098502.086505:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 910
00000100:00100000:2.0:1766098502.086520:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1146
00000100:00100000:2.0:1766098502.086523:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432869440:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766098502.086634:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098502.086667:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432869440:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (443us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098502.086674:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1146
00000100:00100000:2.0:1766098502.086692:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869504
00000100:00100000:2.0:1766098502.086715:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1897
00000100:00100000:2.0:1766098502.086718:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000667f5d6e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432869504:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.086749:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000667f5d6e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432869504:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (449us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098502.086754:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1897
00010000:00080000:2.0:1766098502.086793:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bb97adba x1851885432869568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098509 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098502.086808:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432869568:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766098502.086843:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098502.086848:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098502.086852:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa6a6aec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432869440:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098502.086888:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432869504:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098502.087605:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e8fcc9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432869376:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098502.087784:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098502.092048:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869312
00000100:00100000:2.0:1766098502.092140:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1293
00000100:00100000:2.0:1766098502.092145:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432869312:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098502.093072:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432869312:12345-0@lo:400:kworker/u8:2.0 Request processed in 923us (5288us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098502.093082:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1293
00000100:00100000:1.0:1766098502.093198:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd66eec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432869312:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098502.525794:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098503.550188:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098504.573710:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098505.602376:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098506.530410:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098506.530448:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098506.530537:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098506.530740:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183552
00000100:00100000:3.0:1766098506.530792:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1898
00000100:00100000:3.0:1766098506.530798:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f4d39d96 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417183552:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098506.530899:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f4d39d96 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417183552:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 100us (450us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098506.530906:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1898
00000100:00100000:0.0:1766098506.531123:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183488
00000100:00100000:0.0:1766098506.531150:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1294
00000100:00100000:0.0:1766098506.531154:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417183488:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098506.531184:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417183488:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 30us (782us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098506.531190:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1294
00000100:00100000:1.0:1766098506.531331:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183616
00000100:00100000:1.0:1766098506.531359:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1147
00000100:00100000:1.0:1766098506.531376:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f72fb6bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183616:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098506.531465:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098506.531532:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f72fb6bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183616:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 155us (994us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098506.531539:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1147
00010000:00010000:0.0:1766098506.626292:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098507.197668:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098507.197675:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098507.197678:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098507.197685:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3010
00000100:00080000:3.0:1766098507.197688:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098507.197706:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098507.197726:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000afdf3ced x1851885432869760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098507.197812:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432869824:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098507.197812:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432869888:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098507.197931:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432869632:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098507.198301:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432869696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098507.199085:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098507.199137:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432869760:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098507.199165:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098507.199199:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869824
00000100:00100000:2.0:1766098507.199241:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1899
00000100:00100000:2.0:1766098507.199246:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025358442 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432869824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098507.199292:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025358442 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432869824:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (209us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098507.199300:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1899
00000100:00100000:2.0:1766098507.199326:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869760
00000100:00100000:2.0:1766098507.199344:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1148
00000100:00100000:2.0:1766098507.199347:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a46965fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432869760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766098507.199439:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098507.199469:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a46965fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432869760:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (303us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098507.199503:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1148
00000100:00080000:2.0:1766098507.199534:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098507.199538:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098507.199541:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432869760:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098507.199569:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432869824:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098507.200093:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098507.200127:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869888
00000100:00100000:3.0:1766098507.200153:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 911
00000100:00100000:3.0:1766098507.200157:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432869888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098507.200190:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432869888:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (100us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098507.200196:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 911
00010000:00080000:3.0:1766098507.200236:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a79ed326 x1851885432869888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098514 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098507.200246:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79ed326 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432869888:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098507.201710:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098507.201736:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098507.201798:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869632
00000100:00100000:3.0:1766098507.201856:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1295
00000100:00100000:3.0:1766098507.201862:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432869632:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098507.201921:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432869632:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (194us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098507.201928:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1295
00000100:00100000:2.0:1766098507.202041:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869696
00000100:00100000:2.0:1766098507.202258:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900
00000100:00100000:2.0:1766098507.202262:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000050b9c6b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432869696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098507.203729:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432869696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098507.204778:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000050b9c6b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432869696:12345-0@lo:400:kworker/u8:2.0 Request processed in 2510us (3068us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098507.204789:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900
00000100:00100000:1.0:1766098507.207273:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432869632:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098507.646934:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098508.670093:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098509.694262:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098510.717745:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098511.644678:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098511.644721:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098511.644750:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183680
00000100:00100000:3.0:1766098511.644756:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098511.644804:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183808
00000100:00100000:1.0:1766098511.644849:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1149
00000100:00100000:1.0:1766098511.644856:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183808:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098511.644954:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098511.645031:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417183808:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 174us (274us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098511.645038:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1149
00000100:00100000:2.0:1766098511.645270:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183744
00000100:00100000:2.0:1766098511.645314:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1901
00000100:00100000:2.0:1766098511.645318:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a79ed326 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417183744:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098511.645368:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a79ed326 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417183744:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 49us (647us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098511.645375:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1901
00000100:00100000:0.0:1766098511.645460:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1296
00000100:00100000:0.0:1766098511.645464:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417183680:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098511.645536:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417183680:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 71us (864us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098511.645562:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1296
00010000:00010000:2.0:1766098511.741633:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098512.317886:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098512.317892:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098512.317896:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098512.317915:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3015
00000100:00080000:2.0:1766098512.317918:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098512.317923:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098512.317943:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b6c1f26a x1851885432870080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098512.317980:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432870144:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098512.318040:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098512.318070:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098512.318075:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432870016:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098512.318096:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098512.318119:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432870080:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098512.318143:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098512.318176:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870144
00000100:00100000:2.0:1766098512.318211:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870016
00000100:00100000:2.0:1766098512.318245:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1902
00000100:00100000:2.0:1766098512.318251:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060c730a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432870144:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098512.318296:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060c730a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432870144:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (256us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098512.318302:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1902
00000100:00100000:2.0:1766098512.318317:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1903
00000100:00100000:2.0:1766098512.318320:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d48a9336 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432870016:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098512.318342:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d48a9336 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432870016:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (246us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098512.318347:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1903
00000100:00100000:2.0:1766098512.318379:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432870016:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098512.318399:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432870144:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098512.318606:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870080
00000100:00100000:3.0:1766098512.318633:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1150
00000100:00100000:3.0:1766098512.318637:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432870080:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098512.318725:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098512.318756:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432870080:12345-0@lo:8:kworker/u8:0.0 Request processed in 118us (613us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098512.318761:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1150
00000100:00080000:3.0:1766098512.318782:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098512.318784:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098512.318797:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432870080:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098512.318813:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432869952:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098512.318838:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098512.318851:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098512.318855:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432870208:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098512.318870:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098512.318888:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432869952
00000100:00100000:3.0:1766098512.318907:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1297
00000100:00100000:3.0:1766098512.318910:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432869952:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098512.318933:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432869952:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (94us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098512.318937:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1297
00000100:00100000:3.0:1766098512.318958:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870208
00000100:00100000:3.0:1766098512.318972:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 912
00000100:00100000:3.0:1766098512.318974:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432870208:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098512.318992:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432870208:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (121us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098512.318997:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 912
00010000:00080000:3.0:1766098512.319028:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b2232645 x1851885432870208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098519 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098512.319037:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432870208:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098512.319045:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432869952:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098512.766831:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098513.789890:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098514.816401:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098515.838456:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098516.767797:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098516.769279:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098516.770043:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183936
00000100:00100000:2.0:1766098516.771467:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1904
00000100:00100000:2.0:1766098516.771487:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c51db5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417183936:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098516.771572:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c51db5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417183936:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 95us (3783us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098516.771579:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1904
00000100:00100000:1.0:1766098516.771910:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098516.772102:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417183872
00000100:00100000:3.0:1766098516.772146:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1298
00000100:00100000:3.0:1766098516.772151:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417183872:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098516.772185:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417183872:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 33us (2913us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098516.772192:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1298
00000100:00100000:2.0:1766098516.773696:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184000
00000100:00100000:2.0:1766098516.773736:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1151
00000100:00100000:2.0:1766098516.773741:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417184000:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098516.773842:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098516.773907:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417184000:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 165us (1998us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098516.773914:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1151
00010000:00010000:2.0:1766098516.865139:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098517.437984:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098517.437992:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:1.0:1766098517.437996:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432870272:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098517.437996:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098517.438005:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3020
00000100:00080000:3.0:1766098517.438008:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098517.438013:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098517.438052:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000369e20f7 x1851885432870400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098517.438067:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098517.438105:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047cad513 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432870528:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098517.438111:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870272
00000100:00100000:3.0:1766098517.438150:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098517.438157:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1299
00000100:00100000:1.0:1766098517.438165:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432870272:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098517.438188:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432870336:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098517.438225:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098517.438234:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432870272:12345-0@lo:400:kworker/u8:2.0 Request processed in 68us (168us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098517.438243:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1299
00000100:00100000:0.0:1766098517.438246:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432870272:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098517.438248:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098517.438258:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432870464:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098517.438281:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870336
00000100:00100000:3.0:1766098517.438295:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098517.438298:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432870400:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098517.438329:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098517.438368:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870528
00000100:00100000:1.0:1766098517.438373:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870400
00000100:00100000:2.0:1766098517.438394:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 913
00000100:00100000:2.0:1766098517.438398:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432870528:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098517.438412:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1152
00000100:00100000:1.0:1766098517.438417:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432870400:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098517.438436:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432870528:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (287us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098517.438442:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 913
00010000:00080000:2.0:1766098517.438500:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000047cad513 x1851885432870528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098524 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098517.438521:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047cad513 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432870528:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766098517.438534:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00080000:2.0:1766098517.438909:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098517.438913:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098517.438918:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432870400:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098517.439522:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432870400:12345-0@lo:8:kworker/u8:2.0 Request processed in 1102us (1191us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098517.439532:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1152
00000100:00100000:1.0:1766098517.440929:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870464
00000100:00100000:1.0:1766098517.440973:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1905
00000100:00100000:1.0:1766098517.440979:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043ef6b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432870336:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098517.441034:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906
00000100:00100000:3.0:1766098517.441040:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8d3efcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432870464:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098517.441053:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043ef6b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432870336:12345-0@lo:400:kworker/u8:2.0 Request processed in 73us (2829us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098517.441060:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1905
00000100:00100000:3.0:1766098517.441084:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8d3efcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432870464:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (2791us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098517.441092:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906
00000100:00100000:1.0:1766098517.441114:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432870464:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098517.441129:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444deb29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432870336:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766098517.888891:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098518.909696:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098519.936920:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098520.962231:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098521.887538:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098521.888967:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184064
00000100:00100000:3.0:1766098521.889034:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1300
00000100:00100000:3.0:1766098521.889042:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417184064:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098521.889120:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417184064:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 78us (1590us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098521.889128:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1300
00000100:00100000:1.0:1766098521.889424:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098521.891183:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184128
00000100:00100000:0.0:1766098521.891266:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1907
00000100:00100000:0.0:1766098521.891272:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df8193b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417184128:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098521.891429:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df8193b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417184128:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 155us (2006us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098521.891437:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1907
00000100:00100000:0.0:1766098521.892968:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098521.893443:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184192
00000100:00100000:1.0:1766098521.893513:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1153
00000100:00100000:1.0:1766098521.893518:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184192:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098521.893609:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098521.893664:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184192:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 144us (701us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098521.893670:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1153
00010000:00010000:0.0:1766098521.981736:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098522.558569:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098522.558577:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098522.558580:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098522.558588:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3025
00000100:00080000:3.0:1766098522.558592:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098522.558597:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098522.558616:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c51052e1 x1851885432870720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098522.558659:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432870656:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098522.558731:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098522.558757:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098522.558763:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432870784:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098522.558802:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098522.558841:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870656
00000100:00100000:3.0:1766098522.558874:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870784
00000100:00100000:3.0:1766098522.558891:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1908
00000100:00100000:3.0:1766098522.558896:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047cad513 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432870656:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098522.558949:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047cad513 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432870656:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (220us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098522.558956:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1908
00000100:00100000:3.0:1766098522.558970:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1909
00000100:00100000:3.0:1766098522.558973:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003fede54a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432870784:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098522.558994:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003fede54a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432870784:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (193us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098522.558999:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1909
00000100:00100000:3.0:1766098522.559034:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432870784:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098522.559043:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432870656:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098522.559120:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432870848:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098522.559150:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098522.559178:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098522.559184:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432870592:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098522.559212:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098522.559237:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c51052e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432870720:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098522.559260:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098522.559288:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870848
00000100:00100000:2.0:1766098522.559326:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870720
00000100:00100000:2.0:1766098522.559343:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 914
00000100:00100000:2.0:1766098522.559348:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432870848:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098522.559387:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432870848:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (236us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098522.559394:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 914
00000100:00100000:2.0:1766098522.559411:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1154
00000100:00100000:2.0:1766098522.559415:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432870720:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098522.559433:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870592
00000100:00100000:0.0:1766098522.559459:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1301
00000100:00100000:0.0:1766098522.559463:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432870592:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098522.559512:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432870592:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (302us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098522.559517:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1301
00010000:02020000:2.0:1766098522.559543:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098522.559588:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432870720:12345-0@lo:8:kworker/u8:0.0 Request processed in 173us (329us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098522.559602:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1154
00000100:00100000:2.0:1766098522.559627:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432870592:0@lo:400:kworker/u8:0.0
00010000:00080000:2.0:1766098522.559637:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000369e20f7 x1851885432870848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098529 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098522.559646:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432870848:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098522.559985:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098522.559989:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098522.559994:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c51052e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432870720:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098523.009801:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098524.033231:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098525.054070:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098526.077740:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098527.011149:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098527.011213:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098527.011267:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098527.011314:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184384
00000100:00100000:2.0:1766098527.011368:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1155
00000100:00100000:2.0:1766098527.011377:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd66eec1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184384:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098527.011517:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098527.011591:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd66eec1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184384:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 213us (323us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098527.011599:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1155
00000100:00100000:3.0:1766098527.011738:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184320
00000100:00100000:3.0:1766098527.011779:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1910
00000100:00100000:3.0:1766098527.011783:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063bda73f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417184320:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098527.011837:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063bda73f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417184320:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 53us (699us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098527.011844:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1910
00000100:00100000:1.0:1766098527.012400:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184256
00000100:00100000:1.0:1766098527.012447:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1302
00000100:00100000:1.0:1766098527.012452:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417184256:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098527.012720:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417184256:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 266us (1507us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098527.012728:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1302
00010000:00010000:1.0:1766098527.101978:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098527.677700:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766098527.677718:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098527.677725:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098527.677729:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1766098527.677732:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000449bd1d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432870912:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098527.677754:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3031
00000100:00080000:3.0:1766098527.677757:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098527.677764:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098527.677790:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000098532044 x1851885432871040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098527.677830:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098527.677848:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432870976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098527.677872:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc97a667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432871168:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098527.677891:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098527.677942:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098527.677943:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098527.677950:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073f4fbbd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432871104:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098527.677979:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098527.677996:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871168
00000100:00100000:3.0:1766098527.678018:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870976
00000100:00100000:3.0:1766098527.678043:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871104
00000100:00100000:2.0:1766098527.678045:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 915
00000100:00100000:2.0:1766098527.678051:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432871168:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098527.678059:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1911
00000100:00100000:3.0:1766098527.678062:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004833a3bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432870976:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098527.678101:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004833a3bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432870976:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (212us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098527.678109:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1911
00000100:00100000:1.0:1766098527.678133:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912
00000100:00100000:1.0:1766098527.678140:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000965f5dd0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432871104:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098527.678180:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000965f5dd0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432871104:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (201us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098527.678186:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432870976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098527.678188:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912
00000100:00100000:3.0:1766098527.678214:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073f4fbbd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432871104:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098527.678286:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432871168:12345-0@lo:400:kworker/u8:0.0 Request processed in 234us (345us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098527.678293:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 915
00000100:00100000:2.0:1766098527.678325:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432870912
00000100:00100000:2.0:1766098527.678367:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1303
00000100:00100000:2.0:1766098527.678371:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432870912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098527.678400:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432870912:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (574us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098527.678406:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1303
00000100:00100000:2.0:1766098527.678745:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000098532044 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432871040:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098527.678777:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:0.0:1766098527.678853:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000dc97a667 x1851885432871168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098534 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098527.678865:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc97a667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432871168:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098527.678874:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000449bd1d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432870912:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098527.678935:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871040
00000100:00100000:3.0:1766098527.678969:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1156
00000100:00100000:3.0:1766098527.678973:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e8fcc9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432871040:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098527.679075:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098527.679116:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e8fcc9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432871040:12345-0@lo:8:kworker/u8:0.0 Request processed in 142us (339us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098527.679129:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1156
00000100:00080000:0.0:1766098527.679734:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098527.679740:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098527.679745:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000098532044 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432871040:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766098528.133723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098529.149782:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098530.173848:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098531.200687:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098532.128968:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098532.129021:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098532.129126:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184448
00000100:00100000:2.0:1766098532.129192:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1304
00000100:00100000:2.0:1766098532.129199:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417184448:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098532.129292:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417184448:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 92us (416us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098532.129299:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1304
00000100:00100000:0.0:1766098532.130240:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098532.130330:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184512
00000100:00100000:1.0:1766098532.130367:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1913
00000100:00100000:1.0:1766098532.130371:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043119846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417184512:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098532.130422:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043119846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417184512:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 50us (1402us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098532.130429:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1913
00000100:00100000:2.0:1766098532.130757:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184576
00000100:00100000:2.0:1766098532.130784:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1157
00000100:00100000:2.0:1766098532.130789:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184576:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098532.131055:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098532.131259:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184576:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 468us (1022us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098532.131266:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1157
00010000:00010000:2.0:1766098532.223081:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098532.798812:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098532.798821:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098532.798825:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098532.798836:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3036
00000100:00080000:3.0:1766098532.798840:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098532.798847:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098532.798872:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000081748fb8 x1851885432871360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098532.798930:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432871296:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799018:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098532.799062:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098532.799068:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000449764d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432871424:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799093:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098532.799137:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871296
00000100:00100000:3.0:1766098532.799184:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871424
00000100:00100000:3.0:1766098532.799201:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1914
00000100:00100000:3.0:1766098532.799207:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6d2bcd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432871296:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799255:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6d2bcd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432871296:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (239us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098532.799262:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1914
00000100:00100000:3.0:1766098532.799277:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1915
00000100:00100000:3.0:1766098532.799280:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000098532044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432871424:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799300:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000098532044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432871424:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (207us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098532.799325:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1915
00000100:00100000:3.0:1766098532.799354:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cb6e8c2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432871488:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799382:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098532.799401:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098532.799406:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073f4fbbd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432871232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799434:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098532.799504:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871232
00000100:00100000:3.0:1766098532.799526:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1305
00000100:00100000:3.0:1766098532.799529:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432871232:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799566:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432871232:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (133us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098532.799572:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1305
00000100:00100000:3.0:1766098532.799764:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073f4fbbd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432871232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799795:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000449764d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432871424:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799803:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432871296:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799822:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081748fb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432871360:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098532.799847:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098532.799878:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871488
00000100:00100000:3.0:1766098532.800707:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871360
00000100:00100000:3.0:1766098532.800733:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 916
00000100:00100000:3.0:1766098532.800737:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000449bd1d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432871488:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098532.800760:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1158
00000100:00100000:1.0:1766098532.800767:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432871360:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098532.800784:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000449bd1d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432871488:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (1401us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098532.800804:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 916
00010000:00080000:3.0:1766098532.800839:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cb6e8c2c x1851885432871488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098539 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098532.800851:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cb6e8c2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432871488:0@lo:400:kworker/u8:0.0
00010000:02020000:1.0:1766098532.800868:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098532.801243:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432871360:12345-0@lo:8:kworker/u8:0.0 Request processed in 474us (1395us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098532.801251:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1158
00000100:00080000:0.0:1766098532.801265:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098532.801270:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098532.801275:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081748fb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432871360:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766098533.246901:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098534.271203:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098535.295932:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098536.317717:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098537.247342:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098537.248277:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098537.248699:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184704
00000100:00100000:2.0:1766098537.248726:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184640
00000100:00100000:2.0:1766098537.248795:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1306
00000100:00100000:2.0:1766098537.248802:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417184640:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098537.248873:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417184640:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 69us (1543us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098537.248881:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1306
00000100:00100000:0.0:1766098537.249315:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1916
00000100:00100000:0.0:1766098537.249321:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b028421 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417184704:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098537.249375:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b028421 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417184704:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 53us (1104us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098537.249382:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1916
00000100:00100000:0.0:1766098537.249890:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098537.249995:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184768
00000100:00100000:3.0:1766098537.250026:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1159
00000100:00100000:3.0:1766098537.250030:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184768:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098537.250133:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098537.251458:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417184768:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1419us (1562us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098537.251471:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1159
00010000:00010000:0.0:1766098537.341735:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098537.917795:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432871552:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.917888:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766098537.918123:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098537.918128:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098537.918131:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098537.918136:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3041
00000100:00080000:0.0:1766098537.918139:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098537.918144:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098537.918168:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003a108df5 x1851885432871680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098537.918224:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003523eb7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432871616:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918253:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098537.918270:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098537.918274:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432871744:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918293:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098537.918325:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871616
00000100:00100000:3.0:1766098537.918361:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871744
00000100:00100000:3.0:1766098537.918380:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1917
00000100:00100000:3.0:1766098537.918385:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb6e8c2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432871616:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918448:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb6e8c2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432871616:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (195us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098537.918455:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1917
00000100:00100000:3.0:1766098537.918471:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918
00000100:00100000:3.0:1766098537.918474:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000449764d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432871744:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918522:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000449764d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432871744:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (229us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098537.918528:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918
00000100:00100000:3.0:1766098537.918569:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432871744:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918578:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003523eb7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432871616:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918610:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098537.918615:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432871808:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918639:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098537.918663:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871808
00000100:00100000:3.0:1766098537.918687:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 917
00000100:00100000:3.0:1766098537.918690:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073f4fbbd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432871808:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918715:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073f4fbbd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432871808:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (76us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098537.918720:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 917
00010000:00080000:3.0:1766098537.918744:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f42693bf x1851885432871808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098544 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098537.918751:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f42693bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432871808:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918781:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432871680:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098537.918814:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098537.918838:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871680
00000100:00100000:3.0:1766098537.918852:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1160
00000100:00100000:3.0:1766098537.918856:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432871680:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098537.918967:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098537.919001:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432871680:12345-0@lo:8:kworker/u8:0.0 Request processed in 144us (187us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098537.919009:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1160
00000100:00080000:3.0:1766098537.919032:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098537.919035:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098537.919038:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432871680:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098537.919567:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871552
00000100:00100000:2.0:1766098537.919617:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1307
00000100:00100000:2.0:1766098537.919621:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432871552:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098537.919658:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432871552:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (1772us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098537.919665:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1307
00000100:00100000:3.0:1766098537.920327:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432871552:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098538.366533:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098539.389740:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098540.416788:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098541.438337:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098542.369199:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098542.369255:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098542.369306:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098542.369438:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184896
00000100:00100000:2.0:1766098542.369443:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184832
00000100:00100000:2.0:1766098542.369549:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1308
00000100:00100000:2.0:1766098542.369558:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417184832:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098542.369575:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1919
00000100:00100000:1.0:1766098542.369586:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417184896:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098542.369668:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417184832:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 108us (360us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098542.369674:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417184896:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 87us (486us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098542.369688:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1308
00000100:00100000:1.0:1766098542.369692:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1919
00000100:00100000:3.0:1766098542.370256:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417184960
00000100:00100000:3.0:1766098542.370603:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1161
00000100:00100000:3.0:1766098542.370609:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417184960:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098542.370713:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098542.370780:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417184960:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 170us (1525us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098542.370788:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1161
00010000:00010000:2.0:1766098542.463365:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098543.037738:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098543.037744:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098543.037748:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098543.037755:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3046
00000100:00080000:1.0:1766098543.037758:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098543.037775:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098543.037793:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000150fb02 x1851885432872000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098543.037878:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432872128:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098543.037900:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078059cfe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432872064:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098543.037950:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098543.037994:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098543.038002:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098543.038013:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006df27fbb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432871872:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098543.038035:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098543.038047:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432871936:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098543.038058:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098543.038101:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871872
00000100:00100000:3.0:1766098543.038115:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098543.038158:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1309
00000100:00100000:3.0:1766098543.038163:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872000:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098543.038169:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432871872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098543.038203:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098543.038242:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432871872:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (189us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098543.038251:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1309
00000100:00100000:3.0:1766098543.038264:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872128
00000100:00100000:0.0:1766098543.038312:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006df27fbb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432871872:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098543.038319:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872000
00000100:00100000:3.0:1766098543.038346:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 918
00000100:00100000:3.0:1766098543.038353:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432872128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098543.038393:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1162
00000100:00100000:0.0:1766098543.038402:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432872000:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098543.038426:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432872128:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (477us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098543.038436:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 918
00010000:00080000:3.0:1766098543.038513:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e485a7e9 x1851885432872128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098550 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00010000:02020000:0.0:1766098543.038526:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098543.038531:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432872128:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098543.038600:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432872000:12345-0@lo:8:kworker/u8:2.0 Request processed in 198us (398us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098543.038611:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1162
00000100:00080000:3.0:1766098543.038619:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098543.038627:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098543.038632:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872000:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098543.039052:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872064
00000100:00100000:1.0:1766098543.039101:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432871936
00000100:00100000:1.0:1766098543.039131:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1920
00000100:00100000:1.0:1766098543.039138:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081748fb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432872064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098543.039197:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081748fb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432872064:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (1204us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098543.039204:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1921
00000100:00100000:1.0:1766098543.039220:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1920
00000100:00100000:2.0:1766098543.039220:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c51052e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432871936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098543.039287:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432871936:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098543.039460:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c51052e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432871936:12345-0@lo:400:kworker/u8:2.0 Request processed in 248us (1346us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098543.039503:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1921
00000100:00100000:1.0:1766098543.039698:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078059cfe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432872064:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098543.486189:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098544.509743:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098545.534671:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098546.564874:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098547.485099:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098547.485161:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098547.485638:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185024
00000100:00100000:3.0:1766098547.485900:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1310
00000100:00100000:3.0:1766098547.485908:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417185024:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098547.485981:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417185024:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (819us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098547.485988:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1310
00000100:00100000:2.0:1766098547.487299:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185088
00000100:00100000:2.0:1766098547.487351:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1922
00000100:00100000:2.0:1766098547.487357:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f42693bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417185088:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098547.487408:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f42693bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417185088:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 51us (2319us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098547.487415:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1922
00000100:00100000:2.0:1766098547.487630:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098547.488756:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185152
00000100:00100000:0.0:1766098547.488995:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1163
00000100:00100000:0.0:1766098547.489001:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417185152:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098547.489096:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098547.489151:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417185152:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 148us (1521us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098547.489159:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1163
00010000:00010000:2.0:1766098547.581767:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098548.158536:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098548.158543:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098548.158560:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098548.158567:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3051
00000100:00080000:3.0:1766098548.158571:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098548.158576:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098548.158598:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000319ec734 x1851885432872320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098548.158646:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872320:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098548.158719:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098548.158765:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872320
00000100:00100000:3.0:1766098548.158797:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1164
00000100:00100000:3.0:1766098548.158803:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052df8ad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432872320:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098548.158898:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098548.158935:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052df8ad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432872320:12345-0@lo:8:kworker/u8:0.0 Request processed in 131us (227us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098548.158943:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1164
00000100:00080000:3.0:1766098548.158977:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098548.158980:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098548.158984:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872320:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098548.159059:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f4bdd76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432872448:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098548.159088:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098548.159114:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098548.159119:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432872192:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098548.159145:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098548.159215:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af55eb76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432872384:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098548.159247:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098548.159274:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098548.159279:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432872256:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098548.159301:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098548.159326:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872448
00000100:00100000:1.0:1766098548.159353:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 919
00000100:00100000:1.0:1766098548.159357:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432872448:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098548.159393:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432872448:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (304us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098548.159398:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 919
00000100:00100000:1.0:1766098548.159417:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872384
00000100:00100000:1.0:1766098548.159451:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872256
00000100:00100000:1.0:1766098548.159502:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1923
00000100:00100000:1.0:1766098548.159506:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432872384:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098548.159550:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432872384:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (303us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098548.159557:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1923
00000100:00100000:1.0:1766098548.159567:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1924
00000100:00100000:1.0:1766098548.159571:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea6f516a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432872256:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098548.159593:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea6f516a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432872256:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (293us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098548.159598:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1924
00000100:00100000:1.0:1766098548.159629:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432872256:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098548.159639:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af55eb76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432872384:0@lo:400:kworker/u8:0.0
00010000:00080000:1.0:1766098548.159665:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009f4bdd76 x1851885432872448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098555 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098548.159674:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f4bdd76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432872448:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098548.160964:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872192
00000100:00100000:3.0:1766098548.161042:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1311
00000100:00100000:3.0:1766098548.161047:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432872192:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098548.162326:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432872192:12345-0@lo:400:kworker/u8:0.0 Request processed in 1274us (3176us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098548.162338:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1311
00000100:00100000:2.0:1766098548.162420:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432872192:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098548.607831:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098549.645621:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098550.655180:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098551.677698:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098552.608303:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098552.608371:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098552.608395:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185216
00000100:00100000:1.0:1766098552.608439:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185344
00000100:00100000:0.0:1766098552.608461:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1312
00000100:00100000:0.0:1766098552.608469:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417185216:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098552.608513:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1165
00000100:00100000:1.0:1766098552.608520:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417185344:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:0.0:1766098552.608604:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417185216:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 135us (309us total) trans 0 rc 0/0
00010000:02020000:1.0:1766098552.608617:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098552.608624:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1312
00000100:00100000:1.0:1766098552.608654:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417185344:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 132us (282us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098552.608662:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1165
00000100:00100000:2.0:1766098552.609559:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098552.609657:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185280
00000100:00100000:0.0:1766098552.609708:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1925
00000100:00100000:0.0:1766098552.609713:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa6a6aec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417185280:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098552.609774:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa6a6aec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417185280:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 60us (223us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098552.609781:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1925
00010000:00010000:0.0:1766098552.701966:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098553.278016:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098553.278024:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098553.278031:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098553.278040:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3056
00000100:00080000:0.0:1766098553.278043:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098553.278049:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098553.278071:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000121c906 x1851885432872640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098553.278125:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001eb4f7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432872768:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278186:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098553.278208:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098553.278213:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432872512:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278239:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098553.278260:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432872576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278286:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098553.278300:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098553.278319:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432872704:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278342:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098553.278371:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872576
00000100:00100000:0.0:1766098553.278405:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872704
00000100:00100000:0.0:1766098553.278423:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1926
00000100:00100000:0.0:1766098553.278428:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072356a30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432872576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278472:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072356a30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432872576:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (187us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098553.278530:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1926
00000100:00100000:0.0:1766098553.278545:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1927
00000100:00100000:0.0:1766098553.278549:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6642cbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432872704:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278574:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6642cbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432872704:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (232us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098553.278579:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1927
00000100:00100000:0.0:1766098553.278611:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432872704:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278621:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432872576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.278660:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872512
00000100:00100000:0.0:1766098553.278682:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1313
00000100:00100000:1.0:1766098553.278822:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872768
00000100:00100000:1.0:1766098553.278854:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 920
00000100:00100000:1.0:1766098553.278867:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432872768:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098553.278905:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432872768:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (720us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098553.278911:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 920
00010000:00080000:1.0:1766098553.278945:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000001eb4f7a x1851885432872768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098560 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766098553.278953:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001eb4f7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432872768:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098553.278980:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000121c906 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872640:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098553.279004:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098553.279025:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872640
00000100:00100000:1.0:1766098553.279039:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1166
00000100:00100000:1.0:1766098553.279042:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432872640:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766098553.279193:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098553.279233:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432872640:12345-0@lo:8:kworker/u8:2.0 Request processed in 189us (227us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098553.279240:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1166
00000100:00080000:1.0:1766098553.279269:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098553.279272:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098553.279275:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000121c906 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098553.279622:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432872512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098553.279676:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432872512:12345-0@lo:400:kworker/u8:2.0 Request processed in 989us (1436us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098553.279683:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1313
00000100:00100000:1.0:1766098553.279945:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432872512:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098553.726699:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098554.749805:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098555.773616:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098556.797734:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098557.724880:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098557.725001:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185408
00000100:00100000:0.0:1766098557.725070:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1314
00000100:00100000:0.0:1766098557.725096:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417185408:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098557.725183:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417185408:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 88us (316us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098557.725192:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1314
00000100:00100000:3.0:1766098557.749823:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098557.749883:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098557.750131:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185472
00000100:00100000:1.0:1766098557.750210:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1928
00000100:00100000:1.0:1766098557.750218:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de7892db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417185472:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098557.750343:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de7892db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417185472:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 125us (533us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098557.750353:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1928
00000100:00100000:2.0:1766098557.751263:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185536
00000100:00100000:2.0:1766098557.751321:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1167
00000100:00100000:2.0:1766098557.751326:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417185536:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098557.751434:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098557.751549:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417185536:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 222us (1666us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098557.751570:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1167
00010000:00010000:2.0:1766098557.822241:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098558.401429:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098558.401439:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098558.401443:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098558.401462:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3061
00000100:00080000:0.0:1766098558.401467:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098558.401495:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098558.401523:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c2e914bf x1851885432872960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098558.401615:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4d1cfad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432873088:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.401693:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098558.401730:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098558.401735:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432872832:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.401808:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098558.401835:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432872896:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.401860:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098558.401874:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098558.401878:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432873024:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.401892:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098558.401914:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2e914bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872960:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098558.401933:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098558.401986:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873088
00000100:00100000:0.0:1766098558.402027:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872960
00000100:00100000:0.0:1766098558.402044:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 921
00000100:00100000:0.0:1766098558.402051:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a431d56b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432873088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.402106:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a431d56b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432873088:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (413us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098558.402114:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 921
00000100:00100000:0.0:1766098558.402134:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1168
00000100:00100000:0.0:1766098558.402138:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095954fe9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432872960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098558.402242:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098558.402271:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095954fe9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432872960:12345-0@lo:8:kworker/u8:0.0 Request processed in 133us (338us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098558.402277:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1168
00000100:00100000:0.0:1766098558.402296:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872832
00000100:00100000:0.0:1766098558.402319:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1315
00000100:00100000:0.0:1766098558.402323:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432872832:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.402347:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432872832:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (586us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098558.402353:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1315
00000100:00100000:0.0:1766098558.402378:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432872896
00000100:00100000:0.0:1766098558.402402:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873024
00000100:00100000:0.0:1766098558.402417:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1929
00000100:00100000:0.0:1766098558.402420:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d9cfcb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432872896:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.402460:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d9cfcb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432872896:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (600us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098558.402466:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1929
00000100:00100000:0.0:1766098558.402511:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1930
00000100:00100000:0.0:1766098558.402515:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d2293fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432873024:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.402541:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d2293fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432873024:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (648us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098558.402546:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1930
00000100:00100000:0.0:1766098558.402583:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432872832:0@lo:400:kworker/u8:0.0
00010000:00080000:0.0:1766098558.402596:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a4d1cfad x1851885432873088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098565 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098558.402605:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4d1cfad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432873088:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098558.402636:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098558.402639:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098558.402643:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2e914bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432872960:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098558.402673:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432873024:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098558.402681:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432872896:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098558.845727:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098559.869720:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098560.893853:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098561.917710:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098562.849536:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098562.849590:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098562.849717:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185728
00000100:00100000:0.0:1766098562.849803:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185664
00000100:00100000:0.0:1766098562.849876:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1931
00000100:00100000:0.0:1766098562.849884:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d1df7f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417185664:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098562.850001:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d1df7f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417185664:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 117us (536us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098562.850010:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1931
00000100:00100000:1.0:1766098562.850241:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1169
00000100:00100000:1.0:1766098562.850251:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417185728:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098562.850349:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098562.851888:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417185728:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1632us (2292us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098562.851901:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1169
00000100:00100000:3.0:1766098562.852435:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098562.852516:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185600
00000100:00100000:0.0:1766098562.852577:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1316
00000100:00100000:0.0:1766098562.852582:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417185600:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098562.852656:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417185600:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 73us (229us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098562.852662:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1316
00010000:00010000:2.0:1766098562.941680:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098563.518540:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1766098563.518540:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766098563.518545:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098563.518549:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098563.518555:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3066
00000100:00080000:2.0:1766098563.518558:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:3.0:1766098563.518559:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432873152:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098563.518563:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098563.518582:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b6c1f26a x1851885432873280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098563.518622:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098563.518653:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020ca4bf7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432873408:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.518658:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873152
00000100:00100000:3.0:1766098563.518677:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098563.518691:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1317
00000100:00100000:2.0:1766098563.518697:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432873152:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.518758:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432873152:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (136us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098563.518766:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1317
00000100:00100000:1.0:1766098563.518792:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873408
00000100:00100000:2.0:1766098563.518814:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432873280:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098563.518831:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 922
00000100:00100000:1.0:1766098563.518837:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432873408:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.518852:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098563.518884:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432873408:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (205us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098563.518888:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a421d01c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432873344:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098563.518894:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 922
00000100:00100000:1.0:1766098563.518917:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873280
00000100:00100000:2.0:1766098563.518928:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098563.518940:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1170
00000100:00100000:2.0:1766098563.518948:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098563.518954:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432873216:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.518992:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766098563.519016:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000020ca4bf7 x1851885432873408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098570 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098563.519018:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873344
00000100:00100000:3.0:1766098563.519027:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020ca4bf7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432873408:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098563.519036:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432873152:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.519040:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873216
00000100:00100000:2.0:1766098563.519055:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1932
00000100:00100000:2.0:1766098563.519059:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432873344:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.519088:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432873344:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (172us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098563.519094:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1932
00000100:00100000:2.0:1766098563.519104:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1933
00000100:00100000:2.0:1766098563.519107:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000532dc8bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432873216:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.519126:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000532dc8bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432873216:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (134us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098563.519131:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1933
00000100:00100000:2.0:1766098563.519152:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432873216:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098563.519160:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a421d01c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432873344:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098563.519815:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432873280:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098563.519903:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098563.519956:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432873280:12345-0@lo:8:kworker/u8:0.0 Request processed in 141us (1118us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098563.519962:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1170
00000100:00080000:0.0:1766098563.520289:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098563.520294:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098563.520298:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432873280:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098563.965660:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098564.991297:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098566.013691:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098567.041909:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098567.968931:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098567.968993:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098567.969026:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098567.969080:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185856
00000100:00100000:2.0:1766098567.969140:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1934
00000100:00100000:2.0:1766098567.969148:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c57c659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417185856:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098567.969237:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c57c659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417185856:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 89us (211us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098567.969261:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1934
00000100:00100000:1.0:1766098567.969395:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185792
00000100:00100000:1.0:1766098567.969436:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1318
00000100:00100000:1.0:1766098567.969440:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417185792:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098567.969471:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417185792:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 31us (479us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098567.969501:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1318
00000100:00100000:3.0:1766098567.969592:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185920
00000100:00100000:3.0:1766098567.969631:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1171
00000100:00100000:3.0:1766098567.969635:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417185920:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098567.969734:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098567.969767:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417185920:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 131us (848us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098567.969773:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1171
00010000:00010000:0.0:1766098568.062530:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098568.638535:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098568.638544:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098568.638549:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098568.638559:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3071
00000100:00080000:2.0:1766098568.638564:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098568.638571:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098568.638596:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b4d81768 x1851885432873600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098568.638656:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432873536:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098568.639089:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432873472:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098568.639497:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432873600:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098568.639535:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098568.639945:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098568.639986:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098568.639993:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432873728:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098568.640023:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098568.640068:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873472
00000100:00100000:1.0:1766098568.640104:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098568.640121:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1319
00000100:00100000:0.0:1766098568.640129:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432873472:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098568.640148:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020ca4bf7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432873664:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098568.640190:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098568.640197:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432873472:12345-0@lo:400:kworker/u8:2.0 Request processed in 67us (251us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098568.640207:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1319
00000100:00100000:1.0:1766098568.640241:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873728
00000100:00100000:0.0:1766098568.640269:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432873472:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098568.640274:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873600
00000100:00100000:1.0:1766098568.640296:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 923
00000100:00100000:1.0:1766098568.640303:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005363ed5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432873728:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098568.640308:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1172
00000100:00100000:0.0:1766098568.640314:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432873600:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098568.640345:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005363ed5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432873728:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (322us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098568.640353:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 923
00010000:02020000:0.0:1766098568.640465:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098568.640523:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432873600:12345-0@lo:8:kworker/u8:2.0 Request processed in 208us (421us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098568.640531:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1172
00010000:00080000:0.0:1766098568.640560:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000061093366 x1851885432873728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098575 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098568.640573:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432873728:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766098568.640628:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098568.640633:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098568.640637:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432873600:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098568.640689:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873536
00000100:00100000:0.0:1766098568.640724:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873664
00000100:00100000:0.0:1766098568.640742:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1935
00000100:00100000:0.0:1766098568.640746:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037d49a48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432873536:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098568.640784:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037d49a48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432873536:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (1252us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098568.640792:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1935
00000100:00100000:0.0:1766098568.640809:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1936
00000100:00100000:0.0:1766098568.640813:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006df27fbb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432873664:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098568.640845:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006df27fbb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432873664:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (656us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098568.640851:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1936
00000100:00100000:0.0:1766098568.640912:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020ca4bf7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432873664:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098568.641029:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432873536:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098569.089005:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098570.110538:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098571.135007:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098572.158941:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098573.089403:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098573.089856:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098573.089935:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098573.090438:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186112
00000100:00100000:2.0:1766098573.090684:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1173
00000100:00100000:2.0:1766098573.090693:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1ae343a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417186112:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098573.090788:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098573.090850:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1ae343a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417186112:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 157us (915us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098573.090857:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1173
00000100:00100000:1.0:1766098573.091029:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417185984
00000100:00100000:0.0:1766098573.091126:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186048
00000100:00100000:0.0:1766098573.091157:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1937
00000100:00100000:0.0:1766098573.091161:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417186048:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098573.091210:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417186048:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 47us (1813us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098573.091217:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1937
00000100:00100000:1.0:1766098573.091270:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1320
00000100:00100000:1.0:1766098573.091276:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417185984:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098573.091320:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417185984:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 44us (1466us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098573.091327:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1320
00010000:00010000:2.0:1766098573.182434:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098573.757875:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432873792:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098573.757892:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098573.757899:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098573.757903:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098573.757909:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3077
00000100:00080000:0.0:1766098573.757913:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098573.757918:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098573.757941:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c2e914bf x1851885432873920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098573.757963:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098573.757997:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4d1cfad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432873984:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098573.758004:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098573.758013:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432874048:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098573.758039:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098573.758054:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098573.758065:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098573.758073:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432873856:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098573.758105:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098573.758108:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873792
00000100:00100000:0.0:1766098573.758134:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2e914bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432873920:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098573.758154:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1321
00000100:00100000:2.0:1766098573.758165:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432873792:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098573.758166:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098573.758199:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874048
00000100:00100000:2.0:1766098573.758220:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432873792:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (257us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098573.758228:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873920
00000100:00100000:2.0:1766098573.758229:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1321
00000100:00100000:0.0:1766098573.758248:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924
00000100:00100000:0.0:1766098573.758252:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432874048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098573.758277:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432873792:0@lo:400:kworker/u8:0.0
00010000:00080000:2.0:1766098573.758304:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008c786030 x1851885432874048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098580 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098573.758320:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432874048:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098573.758369:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873984
00000100:00100000:2.0:1766098573.758377:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1174
00000100:00100000:2.0:1766098573.758383:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a75c5a33 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432873920:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098573.758399:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432873856
00000100:00100000:3.0:1766098573.758417:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1938
00000100:00100000:3.0:1766098573.758421:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4b5a493 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432873984:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098573.758469:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432874048:12345-0@lo:400:kworker/u8:0.0 Request processed in 215us (419us total) trans 0 rc 0/0
00010000:02020000:2.0:1766098573.758497:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098573.758498:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924
00000100:00100000:0.0:1766098573.758534:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1939
00000100:00100000:0.0:1766098573.758541:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000401ac69f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432873856:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098573.758541:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a75c5a33 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432873920:12345-0@lo:8:kworker/u8:0.0 Request processed in 156us (379us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098573.758556:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1174
00000100:00080000:2.0:1766098573.758588:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098573.758591:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098573.758595:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2e914bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432873920:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098573.758640:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4b5a493 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432873984:12345-0@lo:400:kworker/u8:0.0 Request processed in 217us (601us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098573.758649:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1938
00000100:00100000:0.0:1766098573.758740:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000401ac69f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432873856:12345-0@lo:400:kworker/u8:0.0 Request processed in 196us (631us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098573.758747:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1939
00000100:00100000:0.0:1766098573.758780:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432873856:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098573.758791:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4d1cfad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432873984:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098574.205741:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098575.229666:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098576.253681:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098577.277755:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098578.204581:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098578.204681:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186176
00000100:00100000:1.0:1766098578.204730:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1322
00000100:00100000:1.0:1766098578.204736:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417186176:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098578.204817:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417186176:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 80us (242us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098578.204825:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1322
00000100:00100000:3.0:1766098578.205362:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098578.205460:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186240
00000100:00100000:0.0:1766098578.205533:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1940
00000100:00100000:0.0:1766098578.205537:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5f4f40e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417186240:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098578.205588:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5f4f40e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417186240:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 50us (227us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098578.205595:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1940
00000100:00100000:3.0:1766098578.206046:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098578.206125:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186304
00000100:00100000:2.0:1766098578.206158:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1175
00000100:00100000:2.0:1766098578.206163:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417186304:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098578.206253:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098578.206294:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417186304:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 130us (249us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098578.206311:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1175
00010000:00010000:2.0:1766098578.301681:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098578.877852:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098578.877860:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098578.877863:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1766098578.877867:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766098578.877871:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3082
00000100:00080000:2.0:1766098578.877874:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098578.877880:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098578.877886:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432874112:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.877901:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000df6e7141 x1851885432874240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098578.877950:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098578.877979:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432874368:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098578.878004:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098578.878034:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076159a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432874176:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878062:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432874240:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098578.878062:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098578.878080:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098578.878087:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007baf6919 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432874304:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878099:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098578.878113:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098578.878140:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874112
00000100:00100000:3.0:1766098578.878149:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874176
00000100:00100000:2.0:1766098578.878185:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1323
00000100:00100000:3.0:1766098578.878187:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874304
00000100:00100000:2.0:1766098578.878192:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432874112:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098578.878213:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1941
00000100:00100000:3.0:1766098578.878218:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157816be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432874176:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878239:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432874112:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (289us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098578.878261:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157816be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432874176:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (200us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098578.878263:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1323
00000100:00100000:3.0:1766098578.878271:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1941
00000100:00100000:3.0:1766098578.878312:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1942
00000100:00100000:2.0:1766098578.878313:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874368
00000100:00100000:3.0:1766098578.878318:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432874304:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878338:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874240
00000100:00100000:3.0:1766098578.878355:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432874304:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (243us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098578.878360:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 925
00000100:00100000:3.0:1766098578.878366:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1942
00000100:00100000:2.0:1766098578.878368:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432874368:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878405:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432874368:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (401us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098578.878413:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007baf6919 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432874304:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878414:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 925
00000100:00100000:2.0:1766098578.878432:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1176
00000100:00100000:3.0:1766098578.878433:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076159a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432874176:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098578.878443:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432874240:12345-0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098578.878463:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000da267b78 x1851885432874368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098585 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098578.878472:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432874368:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098578.878514:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432874112:0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766098578.878555:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098578.878596:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432874240:12345-0@lo:8:kworker/u8:0.0 Request processed in 153us (499us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098578.878602:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1176
00000100:00080000:3.0:1766098578.878709:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098578.878712:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098578.878716:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432874240:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098579.332082:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098580.349661:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098581.373680:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098582.398168:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098583.326757:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098583.326804:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098583.326845:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098583.327669:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186496
00000100:00100000:0.0:1766098583.327729:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1177
00000100:00100000:0.0:1766098583.327735:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417186496:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098583.327834:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098583.327895:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417186496:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 159us (1091us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098583.327916:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1177
00000100:00100000:3.0:1766098583.328107:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186368
00000100:00100000:3.0:1766098583.328148:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1324
00000100:00100000:3.0:1766098583.328152:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417186368:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098583.328191:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417186368:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 38us (1443us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098583.328197:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1324
00000100:00100000:1.0:1766098583.331312:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186432
00000100:00100000:1.0:1766098583.331380:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1943
00000100:00100000:1.0:1766098583.331386:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2deff23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417186432:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098583.331474:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2deff23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417186432:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 86us (4628us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098583.331495:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1943
00010000:00010000:0.0:1766098583.421672:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098584.006054:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000742bc8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432874496:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098584.006119:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098584.006202:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432874432:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098584.006236:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098584.006283:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874432
00000100:00100000:0.0:1766098584.006330:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1325
00000100:00100000:0.0:1766098584.006336:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432874432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098584.006376:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432874432:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (140us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098584.006383:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1325
00000100:00100000:0.0:1766098584.006432:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432874432:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098584.006568:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874496
00000100:00100000:1.0:1766098584.006610:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944
00000100:00100000:1.0:1766098584.006614:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432874496:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098584.006687:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098584.006692:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098584.006707:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098584.006712:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3087
00000100:00080000:3.0:1766098584.006714:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098584.006718:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098584.006739:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000cc29c393 x1851885432874560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098584.006972:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432874624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098584.006999:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098584.007029:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874624
00000100:00100000:0.0:1766098584.007051:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1945
00000100:00100000:0.0:1766098584.007055:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4d1cfad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432874624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098584.007086:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4d1cfad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432874624:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (87us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098584.007092:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1945
00000100:00100000:0.0:1766098584.007119:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432874624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098584.007143:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc29c393 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432874560:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098584.007170:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098584.007211:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874560
00000100:00100000:0.0:1766098584.007239:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1178
00000100:00100000:0.0:1766098584.007243:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432874560:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098584.007348:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098584.007383:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432874560:12345-0@lo:8:kworker/u8:0.0 Request processed in 138us (213us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098584.007389:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1178
00000100:00080000:0.0:1766098584.007414:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098584.007417:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098584.007420:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc29c393 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432874560:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098584.007688:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432874496:12345-0@lo:400:kworker/u8:0.0 Request processed in 1072us (1570us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098584.007695:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944
00000100:00100000:1.0:1766098584.007829:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052df8ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432874688:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098584.007854:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098584.007933:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874688
00000100:00100000:3.0:1766098584.007971:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 926
00000100:00100000:3.0:1766098584.007976:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432874688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098584.008015:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432874688:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (160us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098584.008021:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 926
00010000:00080000:3.0:1766098584.008047:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000052df8ad1 x1851885432874688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098591 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098584.008057:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000052df8ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432874688:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098584.008151:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000742bc8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432874496:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098584.448362:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098585.469844:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098586.493782:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098587.518001:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098588.448079:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098588.448130:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098588.448167:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098588.448207:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186688
00000100:00100000:3.0:1766098588.448274:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1179
00000100:00100000:3.0:1766098588.448281:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417186688:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098588.448375:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098588.448431:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417186688:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 149us (263us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098588.448438:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1179
00000100:00100000:0.0:1766098588.448579:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186560
00000100:00100000:0.0:1766098588.448625:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1326
00000100:00100000:0.0:1766098588.448629:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417186560:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098588.448662:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417186560:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 32us (532us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098588.448668:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1326
00000100:00100000:1.0:1766098588.449152:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186624
00000100:00100000:1.0:1766098588.449193:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1946
00000100:00100000:1.0:1766098588.449198:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052df8ad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417186624:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098588.449243:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052df8ad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417186624:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 45us (1171us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098588.449250:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1946
00010000:00010000:2.0:1766098588.542996:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098589.119399:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098589.119405:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098589.119408:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098589.119415:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3092
00000100:00080000:0.0:1766098589.119418:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098589.119423:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098589.119441:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a9b941c6 x1851885432874880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098589.119498:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000744be049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432874944:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098589.119569:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098589.119596:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098589.119600:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432874816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098589.119621:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098589.119639:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9b941c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432874880:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098589.119657:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098589.119685:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874944
00000100:00100000:0.0:1766098589.119715:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874816
00000100:00100000:0.0:1766098589.119729:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1947
00000100:00100000:0.0:1766098589.119733:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d197da08 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432874944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098589.119771:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d197da08 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432874944:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (202us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098589.119776:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1947
00000100:00100000:0.0:1766098589.119795:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1948
00000100:00100000:0.0:1766098589.119798:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000056855bda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432874816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098589.119820:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000056855bda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432874816:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (198us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098589.119826:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1948
00000100:00100000:0.0:1766098589.119860:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432874816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098589.119870:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000744be049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432874944:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098589.119951:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011f37470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875008:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098589.119983:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098589.120012:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098589.120017:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000663baa05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432874752:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098589.120041:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098589.120171:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874752
00000100:00100000:3.0:1766098589.120199:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1327
00000100:00100000:3.0:1766098589.120203:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432874752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098589.120250:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432874752:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (208us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098589.120256:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1327
00000100:00100000:3.0:1766098589.120292:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432874880
00000100:00100000:3.0:1766098589.120307:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1180
00000100:00100000:3.0:1766098589.120311:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432874880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098589.120400:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098589.120431:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432874880:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (773us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098589.120438:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1180
00000100:00100000:3.0:1766098589.120448:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875008
00000100:00100000:3.0:1766098589.120469:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 927
00000100:00100000:3.0:1766098589.120472:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a90eed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098589.120521:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a90eed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875008:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (537us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098589.120527:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 927
00000100:00080000:3.0:1766098589.120553:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098589.120557:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098589.120561:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9b941c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432874880:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098589.120581:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000663baa05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432874752:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766098589.120590:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000011f37470 x1851885432875008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098596 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098589.120612:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011f37470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875008:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098589.567723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098590.589639:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098591.617223:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098592.637683:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098593.565837:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098593.566081:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098593.566180:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098593.566326:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186816
00000100:00100000:1.0:1766098593.566388:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1949
00000100:00100000:1.0:1766098593.566399:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000744be049 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417186816:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098593.566471:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000744be049 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417186816:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (390us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098593.566510:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1949
00000100:00100000:3.0:1766098593.566644:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186752
00000100:00100000:3.0:1766098593.566680:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1328
00000100:00100000:3.0:1766098593.566685:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417186752:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098593.566718:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417186752:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 32us (889us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098593.566725:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1328
00000100:00100000:2.0:1766098593.567847:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186880
00000100:00100000:2.0:1766098593.567894:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1181
00000100:00100000:2.0:1766098593.567898:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417186880:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098593.567989:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098593.568052:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417186880:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 153us (1872us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098593.568059:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1181
00010000:00010000:2.0:1766098593.661678:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098594.237749:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098594.237757:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098594.237761:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766098594.237761:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:0.0:1766098594.237770:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3097
00000100:00080000:0.0:1766098594.237774:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:2.0:1766098594.237776:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011f37470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432875136:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098594.237781:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098594.237806:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000663baa05 x1851885432875200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098594.237844:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098594.237884:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9b941c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432875264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098594.237906:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098594.237957:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2e914bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098594.238033:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098594.238053:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098594.238058:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffecbb46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432875072:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098594.238084:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098594.238107:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000663baa05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432875200:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098594.238128:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098594.238161:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875328
00000100:00100000:3.0:1766098594.238209:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875200
00000100:00100000:3.0:1766098594.238232:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 928
00000100:00100000:3.0:1766098594.238238:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098594.238263:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875136
00000100:00100000:3.0:1766098594.238291:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875328:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (298us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098594.238295:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875264
00000100:00100000:3.0:1766098594.238300:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 928
00000100:00100000:0.0:1766098594.238318:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1950
00000100:00100000:3.0:1766098594.238323:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1182
00000100:00100000:0.0:1766098594.238325:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de0eb61b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432875136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098594.238331:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432875200:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098594.238367:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de0eb61b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432875136:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (526us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098594.238373:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1950
00000100:00100000:0.0:1766098594.238391:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1951
00000100:00100000:0.0:1766098594.238396:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014d07368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432875264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098594.238403:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011f37470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432875136:0@lo:400:kworker/u8:0.0
00010000:02020000:3.0:1766098594.238426:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098594.238430:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9b941c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432875264:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098594.238434:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014d07368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432875264:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (526us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098594.238443:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1951
00000100:00100000:3.0:1766098594.238497:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432875200:12345-0@lo:8:kworker/u8:0.0 Request processed in 146us (347us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098594.238504:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1182
00000100:00100000:0.0:1766098594.238513:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875072
00000100:00100000:0.0:1766098594.238543:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1329
00000100:00100000:0.0:1766098594.238547:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432875072:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098594.238595:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098594.238600:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098594.238606:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000663baa05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432875200:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098594.238608:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432875072:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (521us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098594.238617:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1329
00000100:00100000:3.0:1766098594.238638:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffecbb46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432875072:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766098594.238653:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c2e914bf x1851885432875328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098601 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098594.238661:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2e914bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875328:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098594.688125:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098595.709709:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098596.733662:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098597.757857:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098598.684714:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098598.685181:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187072
00000100:00100000:0.0:1766098598.685800:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1183
00000100:00100000:0.0:1766098598.685807:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417187072:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098598.685904:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098598.685977:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417187072:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 169us (1270us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098598.685984:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1183
00000100:00100000:1.0:1766098598.686370:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098598.686512:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187008
00000100:00100000:2.0:1766098598.686519:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098598.686551:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417186944
00000100:00100000:1.0:1766098598.686556:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1952
00000100:00100000:1.0:1766098598.686562:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417187008:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098598.686581:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1330
00000100:00100000:2.0:1766098598.686587:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417186944:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098598.686609:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417187008:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 45us (240us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098598.686618:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417186944:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 29us (99us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098598.686619:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1952
00000100:00100000:2.0:1766098598.686629:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1330
00010000:00010000:0.0:1766098598.784779:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098599.357839:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098599.357846:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098599.357849:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098599.357858:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3102
00000100:00100000:3.0:1766098599.357858:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:1.0:1766098599.357861:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098599.357867:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098599.357876:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432875392:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098599.357886:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e445b90d x1851885432875520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098599.357931:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432875456:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098599.357939:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098599.357968:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098599.357968:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063ac9a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875648:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098599.357989:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098599.357996:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098599.358007:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432875584:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358031:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098599.358042:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432875520:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358064:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875392
00000100:00100000:3.0:1766098599.358068:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098599.358093:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875648
00000100:00100000:1.0:1766098599.358104:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1331
00000100:00100000:1.0:1766098599.358109:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432875392:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098599.358116:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875520
00000100:00100000:3.0:1766098599.358130:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 929
00000100:00100000:3.0:1766098599.358134:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875648:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358162:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432875392:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (223us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098599.358169:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1331
00000100:00100000:3.0:1766098599.358172:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875648:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (177us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098599.358177:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 929
00000100:00100000:3.0:1766098599.358193:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1184
00000100:00100000:3.0:1766098599.358196:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f4bdd76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432875520:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358198:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875456
00000100:00100000:1.0:1766098599.358219:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875584
00000100:00100000:1.0:1766098599.358238:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1953
00000100:00100000:1.0:1766098599.358242:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040addeb2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432875456:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358274:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040addeb2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432875456:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (307us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098599.358280:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1953
00010000:02020000:3.0:1766098599.358286:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098599.358292:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1954
00000100:00100000:1.0:1766098599.358294:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000449bd1d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432875584:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098599.358316:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f4bdd76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432875520:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (249us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098599.358318:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000449bd1d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432875584:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (286us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098599.358322:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1184
00000100:00100000:1.0:1766098599.358323:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1954
00000100:00080000:3.0:1766098599.358356:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:1.0:1766098599.358357:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432875584:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766098599.358359:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098599.358362:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432875520:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358366:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432875456:0@lo:400:kworker/u8:2.0
00010000:00080000:1.0:1766098599.358389:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000063ac9a3f x1851885432875648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098606 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766098599.358396:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063ac9a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875648:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098599.358413:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432875392:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098599.805749:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098600.829860:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098601.853700:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098602.877766:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098603.804796:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098603.804854:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098603.804925:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187264
00000100:00100000:0.0:1766098603.804974:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1185
00000100:00100000:0.0:1766098603.804982:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417187264:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098603.805095:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098603.805206:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417187264:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 223us (351us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098603.805214:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1185
00000100:00100000:2.0:1766098603.805342:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187136
00000100:00100000:2.0:1766098603.805395:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1332
00000100:00100000:2.0:1766098603.805400:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417187136:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098603.805956:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417187136:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 553us (1166us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098603.805964:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1332
00000100:00100000:1.0:1766098603.807267:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098603.807687:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187200
00000100:00100000:3.0:1766098603.807742:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1955
00000100:00100000:3.0:1766098603.807748:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073f4fbbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417187200:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098603.807834:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073f4fbbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417187200:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 85us (571us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098603.807841:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1955
00010000:00010000:2.0:1766098603.902017:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098604.477734:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098604.477741:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098604.477744:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098604.477752:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3107
00000100:00080000:0.0:1766098604.477755:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098604.477760:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098604.477780:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008050eca6 x1851885432875840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098604.477846:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a431d56b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432875904:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.477904:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098604.477933:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098604.477938:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432875776:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.477958:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098604.477981:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008050eca6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432875840:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098604.478005:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098604.478036:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875904
00000100:00100000:0.0:1766098604.478072:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875776
00000100:00100000:0.0:1766098604.478089:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1956
00000100:00100000:0.0:1766098604.478094:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000121c906 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432875904:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.478151:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000121c906 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432875904:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (248us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098604.478158:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1956
00000100:00100000:0.0:1766098604.478177:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1957
00000100:00100000:0.0:1766098604.478181:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001eb4f7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432875776:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.478204:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001eb4f7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432875776:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (245us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098604.478209:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1957
00000100:00100000:0.0:1766098604.478239:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432875776:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.478248:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a431d56b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432875904:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.478272:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875840
00000100:00100000:0.0:1766098604.478289:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1186
00000100:00100000:0.0:1766098604.478293:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e914bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432875840:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098604.478406:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098604.478434:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e914bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432875840:12345-0@lo:8:kworker/u8:0.0 Request processed in 140us (430us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098604.478441:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1186
00000100:00080000:0.0:1766098604.478461:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098604.478464:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098604.478467:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008050eca6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432875840:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098604.478570:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095954fe9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875968:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098604.478597:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098604.478621:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098604.478626:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432875712:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098604.478655:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098604.478686:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875712
00000100:00100000:1.0:1766098604.478707:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1333
00000100:00100000:1.0:1766098604.478711:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432875712:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098604.478739:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432875712:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (84us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098604.478745:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1333
00000100:00100000:1.0:1766098604.478783:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c786030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432875712:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098604.478956:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432875968
00000100:00100000:1.0:1766098604.479555:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 930
00000100:00100000:1.0:1766098604.479561:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875968:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098604.480294:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432875968:12345-0@lo:400:kworker/u8:0.0 Request processed in 730us (1694us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098604.480303:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 930
00010000:00080000:3.0:1766098604.480410:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000095954fe9 x1851885432875968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098611 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098604.480423:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095954fe9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432875968:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098604.928763:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098605.949672:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098606.973828:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098607.997816:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098608.926161:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098608.926368:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187456
00000100:00100000:1.0:1766098608.926419:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1187
00000100:00100000:1.0:1766098608.926426:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417187456:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098608.926553:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098608.926625:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417187456:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 198us (471us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098608.926633:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1187
00000100:00100000:0.0:1766098608.927747:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098608.929051:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187392
00000100:00100000:1.0:1766098608.929126:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1958
00000100:00100000:1.0:1766098608.929287:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a431d56b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417187392:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098608.929353:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a431d56b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417187392:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 67us (1610us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098608.929361:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1958
00000100:00100000:2.0:1766098608.930025:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098608.930283:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187328
00000100:00100000:1.0:1766098608.930324:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1334
00000100:00100000:1.0:1766098608.930329:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417187328:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098608.930385:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417187328:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 55us (361us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098608.930392:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1334
00010000:00010000:2.0:1766098609.022354:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098609.599894:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098609.599903:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098609.599907:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098609.599914:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3112
00000100:00080000:2.0:1766098609.599917:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098609.599938:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098609.599962:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000da267b78 x1851885432876160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098609.600010:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007baf6919 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432876288:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600099:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098609.600130:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098609.600137:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432876032:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600162:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098609.600188:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432876096:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600216:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098609.600232:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098609.600237:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076159a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432876224:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600258:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098609.600290:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876032
00000100:00100000:2.0:1766098609.600326:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1335
00000100:00100000:2.0:1766098609.600331:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432876032:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600376:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432876032:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (213us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098609.600383:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1335
00000100:00100000:2.0:1766098609.600412:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876096
00000100:00100000:2.0:1766098609.600434:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876224
00000100:00100000:2.0:1766098609.600450:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1959
00000100:00100000:2.0:1766098609.600453:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432876096:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600512:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432876096:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (296us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098609.600520:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1959
00000100:00100000:2.0:1766098609.600542:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1960
00000100:00100000:2.0:1766098609.600546:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002288850c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432876224:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600573:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002288850c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432876224:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (316us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098609.600579:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1960
00000100:00100000:2.0:1766098609.600614:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076159a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432876224:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600633:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432876096:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098609.600662:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432876032:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098609.600811:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432876160:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098609.600844:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098609.600891:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876288
00000100:00100000:0.0:1766098609.600925:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876160
00000100:00100000:0.0:1766098609.600943:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 931
00000100:00100000:0.0:1766098609.600947:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432876288:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098609.600980:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432876288:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (883us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098609.600985:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 931
00000100:00100000:0.0:1766098609.601000:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1188
00000100:00100000:0.0:1766098609.601003:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432876160:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098609.601106:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098609.601134:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432876160:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (291us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098609.601148:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1188
00000100:00080000:0.0:1766098609.601186:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098609.601190:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098609.601194:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432876160:0@lo:8:kworker/u8:0.0
00010000:00080000:2.0:1766098609.602921:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007baf6919 x1851885432876288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098616 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098609.602938:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007baf6919 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432876288:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098610.045656:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098611.072503:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098612.093842:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098613.121579:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098614.046677:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098614.046780:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098614.046892:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187584
00000100:00100000:1.0:1766098614.046939:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1961
00000100:00100000:1.0:1766098614.046945:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffecbb46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417187584:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098614.047023:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffecbb46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417187584:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (242us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098614.047039:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1961
00000100:00100000:2.0:1766098614.047227:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187520
00000100:00100000:2.0:1766098614.047259:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1336
00000100:00100000:2.0:1766098614.047264:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417187520:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098614.047301:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417187520:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 37us (632us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098614.047313:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1336
00000100:00100000:0.0:1766098614.049954:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098614.050048:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187648
00000100:00100000:3.0:1766098614.050093:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1189
00000100:00100000:3.0:1766098614.050099:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000663baa05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417187648:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098614.050200:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098614.050246:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000663baa05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417187648:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 146us (300us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098614.050266:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1189
00010000:00010000:0.0:1766098614.143621:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098614.717676:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098614.717682:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098614.717686:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098614.717694:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3118
00000100:00080000:2.0:1766098614.717710:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098614.717716:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098614.717716:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098614.717736:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002684e04b x1851885432876480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098614.717736:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432876352:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.717813:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432876544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098614.717820:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098614.717852:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432876608:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.717861:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098614.717885:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098614.717890:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098614.717893:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432876416:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.717921:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098614.717935:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876352
00000100:00100000:2.0:1766098614.717948:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432876480:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098614.717971:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098614.717980:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1337
00000100:00100000:3.0:1766098614.717987:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432876352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098614.718057:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432876352:12345-0@lo:400:kworker/u8:0.0 Request processed in 70us (239us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098614.718065:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1337
00000100:00100000:3.0:1766098614.718099:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876608
00000100:00100000:3.0:1766098614.718121:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876480
00000100:00100000:3.0:1766098614.718152:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 932
00000100:00100000:3.0:1766098614.718156:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432876608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.718294:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876544
00000100:00100000:2.0:1766098614.718317:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876416
00000100:00100000:2.0:1766098614.718333:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1962
00000100:00100000:2.0:1766098614.718337:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b17b5997 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432876544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098614.718354:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432876608:12345-0@lo:400:kworker/u8:0.0 Request processed in 196us (467us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098614.718362:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 932
00000100:00100000:2.0:1766098614.718378:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b17b5997 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432876544:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (517us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098614.718382:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1190
00000100:00100000:3.0:1766098614.718388:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432876480:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098614.718391:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1962
00000100:00100000:2.0:1766098614.718406:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1963
00000100:00100000:2.0:1766098614.718409:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a03d8323 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432876416:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.718428:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a03d8323 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432876416:12345-0@lo:400:kworker/u8:0.0 Request processed in 18us (508us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098614.718433:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1963
00000100:00100000:2.0:1766098614.718463:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432876416:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.718472:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432876544:0@lo:400:kworker/u8:0.0
00010000:02020000:3.0:1766098614.718514:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:2.0:1766098614.718553:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008834c3ad x1851885432876608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098621 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098614.718555:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432876480:12345-0@lo:8:kworker/u8:0.0 Request processed in 165us (583us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098614.718569:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432876608:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098614.718583:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432876352:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098614.718584:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1190
00000100:00080000:3.0:1766098614.718623:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098614.718627:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098614.718632:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432876480:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098615.166543:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098616.189720:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098617.215430:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098618.237681:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098619.172967:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098619.174405:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098619.174986:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187840
00000100:00100000:2.0:1766098619.175065:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1191
00000100:00100000:2.0:1766098619.175071:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417187840:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098619.175162:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098619.175236:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417187840:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 163us (2275us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098619.175244:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1191
00000100:00100000:0.0:1766098619.175869:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098619.175986:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187776
00000100:00100000:1.0:1766098619.176070:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1964
00000100:00100000:1.0:1766098619.176075:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011f37470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417187776:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098619.176133:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011f37470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417187776:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 57us (1733us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098619.176139:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1964
00000100:00100000:2.0:1766098619.176383:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187712
00000100:00100000:2.0:1766098619.176458:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1338
00000100:00100000:2.0:1766098619.176462:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417187712:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098619.176862:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417187712:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 397us (993us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098619.176870:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1338
00010000:00010000:2.0:1766098619.262682:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098619.838534:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098619.838541:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098619.838543:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098619.838568:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3123
00000100:00080000:2.0:1766098619.838571:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098619.838576:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098619.838595:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098619.838597:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000cbff49d7 x1851885432876800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098619.838611:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432876672:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098619.838653:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432876864:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098619.838866:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098619.838874:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fb915dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432876928:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098619.839190:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432876736:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098619.839253:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098619.839297:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432876800:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098619.839324:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098619.839358:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876800
00000100:00100000:2.0:1766098619.839387:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1192
00000100:00100000:2.0:1766098619.839393:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432876800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098619.839501:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098619.839539:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432876800:12345-0@lo:8:kworker/u8:0.0 Request processed in 145us (215us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098619.839546:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1192
00000100:00080000:2.0:1766098619.839578:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098619.839581:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098619.839584:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432876800:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098619.839614:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876864
00000100:00100000:2.0:1766098619.839641:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1965
00000100:00100000:2.0:1766098619.839645:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071bbd8cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432876864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098619.839676:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098619.839706:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071bbd8cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432876864:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (453us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098619.839714:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1965
00000100:00100000:3.0:1766098619.839716:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876672
00000100:00100000:3.0:1766098619.839750:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1339
00000100:00100000:3.0:1766098619.839755:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432876672:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098619.839775:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432876864:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098619.839796:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432876672:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (120us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098619.839804:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1339
00000100:00100000:2.0:1766098619.839826:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432876672:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098619.840472:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098619.841098:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098619.841212:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876736
00000100:00100000:3.0:1766098619.841258:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1966
00000100:00100000:3.0:1766098619.841264:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432876736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098619.842248:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432876736:12345-0@lo:400:kworker/u8:0.0 Request processed in 980us (1148us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098619.842259:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1966
00000100:00100000:2.0:1766098619.842411:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876928
00000100:00100000:2.0:1766098619.842441:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 933
00000100:00100000:2.0:1766098619.842444:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095954fe9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432876928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098619.842500:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095954fe9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432876928:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (2031us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098619.842505:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 933
00010000:00080000:0.0:1766098619.842684:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006fb915dd x1851885432876928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098626 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098619.842697:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fb915dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432876928:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098619.842852:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432876736:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098620.285851:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098621.310737:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098622.334054:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098623.359840:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098624.285588:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098624.285645:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098624.286228:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187904
00000100:00100000:1.0:1766098624.286290:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1340
00000100:00100000:1.0:1766098624.286297:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417187904:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098624.286375:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417187904:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 77us (795us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098624.286384:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1340
00000100:00100000:2.0:1766098624.286502:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417187968
00000100:00100000:2.0:1766098624.286549:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1967
00000100:00100000:2.0:1766098624.286554:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000742bc8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417187968:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098624.286591:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000742bc8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417187968:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 36us (946us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098624.286598:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1967
00000100:00100000:1.0:1766098624.290571:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098624.290653:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188032
00000100:00100000:2.0:1766098624.290698:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1193
00000100:00100000:2.0:1766098624.290705:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417188032:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098624.290804:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098624.292152:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417188032:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1442us (1582us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098624.292166:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1193
00010000:00010000:2.0:1766098624.381705:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098624.957895:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098624.957902:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098624.957906:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098624.957914:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3128
00000100:00080000:3.0:1766098624.957931:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098624.957936:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098624.957970:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b4243e80 x1851885432877120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098624.958026:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432877056:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.958082:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098624.958107:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098624.958111:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432877184:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.958126:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098624.958146:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4243e80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432877120:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098624.958164:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098624.958197:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877056
00000100:00100000:3.0:1766098624.958230:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877184
00000100:00100000:3.0:1766098624.958244:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1968
00000100:00100000:3.0:1766098624.958249:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd66eec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432877056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.958288:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd66eec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432877056:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (207us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098624.958294:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1968
00000100:00100000:3.0:1766098624.958308:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1969
00000100:00100000:3.0:1766098624.958311:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e8fcc9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432877184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.958336:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e8fcc9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432877184:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (209us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098624.958341:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1969
00000100:00100000:3.0:1766098624.958381:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432877184:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.958391:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432877056:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098624.958496:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc29c393 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432876992:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098624.958530:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098624.958556:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098624.958561:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432877248:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098624.958592:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098624.958618:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877120
00000100:00100000:0.0:1766098624.958638:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877248
00000100:00100000:0.0:1766098624.958656:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 934
00000100:00100000:0.0:1766098624.958660:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c786030 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432877248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098624.958693:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c786030 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432877248:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (100us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098624.958699:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 934
00000100:00100000:0.0:1766098624.958721:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1194
00000100:00100000:0.0:1766098624.958725:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432877120:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098624.958820:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098624.958850:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432877120:12345-0@lo:8:kworker/u8:2.0 Request processed in 124us (686us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098624.958856:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1194
00000100:00080000:0.0:1766098624.958879:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098624.958882:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098624.958885:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4243e80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432877120:0@lo:8:kworker/u8:2.0
00010000:00080000:0.0:1766098624.958904:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004dbc67d9 x1851885432877248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098631 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098624.958912:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432877248:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.959128:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432876992
00000100:00100000:3.0:1766098624.959164:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1341
00000100:00100000:3.0:1766098624.959168:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432876992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098624.959201:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432876992:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (671us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098624.959207:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1341
00000100:00100000:0.0:1766098624.959283:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc29c393 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432876992:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098625.407275:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098626.433744:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098627.454126:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098628.477748:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098629.407147:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098629.408113:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098629.408676:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188096
00000100:00100000:1.0:1766098629.408740:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1342
00000100:00100000:1.0:1766098629.408746:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417188096:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098629.408815:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417188096:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 69us (1677us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098629.408823:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1342
00000100:00100000:0.0:1766098629.409300:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098629.409407:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188160
00000100:00100000:2.0:1766098629.409441:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1970
00000100:00100000:2.0:1766098629.409445:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006f2f7702 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417188160:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098629.409500:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006f2f7702 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417188160:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 53us (1391us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098629.409506:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1970
00000100:00100000:1.0:1766098629.409889:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188224
00000100:00100000:1.0:1766098629.409932:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1195
00000100:00100000:1.0:1766098629.409937:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417188224:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098629.410045:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098629.410893:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417188224:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 952us (1591us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098629.410903:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1195
00010000:00010000:0.0:1766098629.501719:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098630.077705:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098630.077713:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098630.077716:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098630.077723:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3133
00000100:00080000:3.0:1766098630.077727:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098630.077732:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098630.077752:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b2232645 x1851885432877440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098630.077798:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432877504:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098630.077858:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098630.077887:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098630.077892:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432877376:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098630.077910:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098630.078028:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432877568:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078060:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098630.078088:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098630.078093:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432877312:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078117:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098630.078148:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877312
00000100:00100000:2.0:1766098630.078185:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1343
00000100:00100000:2.0:1766098630.078190:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432877312:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078230:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432877312:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (113us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098630.078238:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1343
00000100:00100000:2.0:1766098630.078266:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877504
00000100:00100000:2.0:1766098630.078285:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877376
00000100:00100000:2.0:1766098630.078298:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1971
00000100:00100000:2.0:1766098630.078301:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432877504:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078333:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4243e80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432877504:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (476us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098630.078355:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1971
00000100:00100000:2.0:1766098630.078366:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1972
00000100:00100000:2.0:1766098630.078369:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc29c393 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432877376:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078392:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc29c393 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432877376:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (482us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098630.078398:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1972
00000100:00100000:2.0:1766098630.078423:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432877312:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078450:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432877376:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098630.078457:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432877504:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098630.078559:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877568
00000100:00100000:3.0:1766098630.078591:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 935
00000100:00100000:3.0:1766098630.078595:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1f26126 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432877568:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098630.078627:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1f26126 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432877568:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (566us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098630.078633:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 935
00010000:00080000:2.0:1766098630.078711:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004dbc67d9 x1851885432877568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098637 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098630.078721:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432877568:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098630.078834:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432877440:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098630.078867:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098630.078963:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877440
00000100:00100000:3.0:1766098630.078990:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1196
00000100:00100000:3.0:1766098630.078994:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038bdd4c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432877440:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098630.079107:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098630.079140:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038bdd4c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432877440:12345-0@lo:8:kworker/u8:2.0 Request processed in 144us (272us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098630.079146:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1196
00000100:00080000:3.0:1766098630.079259:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098630.079263:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098630.079267:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432877440:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766098630.528512:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098631.551340:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098632.575349:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098633.597801:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098634.526705:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098634.528392:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188288
00000100:00100000:2.0:1766098634.528472:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1344
00000100:00100000:2.0:1766098634.528511:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417188288:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098634.528599:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417188288:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 88us (1947us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098634.528607:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1344
00000100:00100000:3.0:1766098634.531182:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098634.531235:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098634.531785:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188352
00000100:00100000:0.0:1766098634.531846:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1973
00000100:00100000:0.0:1766098634.531853:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1ae343a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417188352:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098634.531952:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1ae343a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417188352:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 99us (779us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098634.531960:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1973
00000100:00100000:1.0:1766098634.532096:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188416
00000100:00100000:1.0:1766098634.532134:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1197
00000100:00100000:1.0:1766098634.532138:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417188416:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098634.532230:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098634.532259:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417188416:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 121us (1025us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098634.532266:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1197
00010000:00010000:2.0:1766098634.622388:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098635.199570:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098635.199577:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098635.199581:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098635.199589:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3138
00000100:00080000:1.0:1766098635.199592:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098635.199599:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098635.199627:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d5337db0 x1851885432877760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098635.199774:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432877696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098635.199831:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432877888:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098635.199843:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098635.199913:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098635.199925:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432877824:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098635.199946:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098635.199970:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098635.199989:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098635.200002:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432877632:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098635.200031:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432877760:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098635.200047:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098635.200088:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098635.200128:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877888
00000100:00100000:1.0:1766098635.200148:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877696
00000100:00100000:2.0:1766098635.200188:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877760
00000100:00100000:1.0:1766098635.200207:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877824
00000100:00100000:2.0:1766098635.200221:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 936
00000100:00100000:2.0:1766098635.200231:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432877888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098635.200245:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1974
00000100:00100000:1.0:1766098635.200252:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432877696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098635.200313:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432877888:12345-0@lo:400:kworker/u8:2.0 Request processed in 81us (368us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098635.200325:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 936
00000100:00100000:1.0:1766098635.200330:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432877696:12345-0@lo:400:kworker/u8:2.0 Request processed in 73us (485us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098635.200343:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1974
00000100:00100000:2.0:1766098635.200358:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1198
00000100:00100000:2.0:1766098635.200364:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432877760:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098635.200372:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1975
00000100:00100000:1.0:1766098635.200378:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432877824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098635.200406:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432877824:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (440us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098635.200411:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1975
00000100:00100000:1.0:1766098635.200457:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432877824:0@lo:400:kworker/u8:2.0
00010000:02020000:2.0:1766098635.200468:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098635.200470:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432877696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098635.200565:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432877760:12345-0@lo:8:kworker/u8:2.0 Request processed in 198us (479us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098635.200571:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877632
00000100:00100000:2.0:1766098635.200576:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1198
00000100:00100000:1.0:1766098635.200619:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1345
00000100:00100000:1.0:1766098635.200629:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432877632:12345-0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766098635.200634:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000039f6716e x1851885432877888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098642 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766098635.200650:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432877888:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098635.200684:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432877632:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098635.200685:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432877632:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (637us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098635.200700:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1345
00000100:00080000:2.0:1766098635.200749:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098635.200755:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098635.200761:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432877760:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766098635.646092:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098636.669814:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098637.694315:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098638.719213:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098639.646751:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098639.646808:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098639.646852:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188544
00000100:00100000:2.0:1766098639.646908:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1976
00000100:00100000:2.0:1766098639.646915:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417188544:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098639.646996:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417188544:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 81us (252us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098639.647003:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1976
00000100:00100000:1.0:1766098639.647104:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188480
00000100:00100000:1.0:1766098639.647155:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1346
00000100:00100000:1.0:1766098639.647160:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417188480:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098639.647188:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417188480:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 28us (382us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098639.647195:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1346
00000100:00100000:2.0:1766098639.651107:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098639.651204:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188608
00000100:00100000:0.0:1766098639.651461:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1199
00000100:00100000:0.0:1766098639.651467:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbff49d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417188608:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098639.651580:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098639.654572:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbff49d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417188608:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 3096us (3464us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098639.654587:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1199
00010000:00010000:2.0:1766098639.742273:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098640.317770:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098640.317777:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098640.317781:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098640.317789:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3143
00000100:00080000:2.0:1766098640.317792:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098640.317797:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098640.317816:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009db79f91 x1851885432878080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098640.317870:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b44b2785 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432878144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.317930:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098640.317959:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098640.317964:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ce9aa68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432878016:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.317986:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098640.318025:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009db79f91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432878080:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098640.318050:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098640.318103:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878080
00000100:00100000:2.0:1766098640.318135:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1200
00000100:00100000:2.0:1766098640.318141:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432878080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766098640.318238:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098640.318293:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432878080:12345-0@lo:8:kworker/u8:2.0 Request processed in 151us (241us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098640.318300:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1200
00000100:00100000:2.0:1766098640.318326:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878144
00000100:00100000:2.0:1766098640.318352:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878016
00000100:00100000:2.0:1766098640.318365:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1977
00000100:00100000:2.0:1766098640.318368:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432878144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.318403:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432878144:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (474us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098640.318408:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1977
00000100:00100000:2.0:1766098640.318419:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1978
00000100:00100000:2.0:1766098640.318422:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007abdfee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432878016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.318445:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007abdfee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432878016:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (460us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098640.318451:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1978
00000100:00080000:2.0:1766098640.318507:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098640.318510:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098640.318514:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009db79f91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432878080:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766098640.318540:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ce9aa68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432878016:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.318547:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b44b2785 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432878144:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098640.319164:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432877952:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098640.319203:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098640.319242:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098640.319247:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432878208:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098640.319269:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098640.320452:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432877952
00000100:00100000:3.0:1766098640.320525:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1347
00000100:00100000:3.0:1766098640.320530:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432877952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098640.320598:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432877952:12345-0@lo:400:kworker/u8:2.0 Request processed in 68us (1396us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098640.320609:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1347
00000100:00100000:0.0:1766098640.321141:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432877952:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.321569:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878208
00000100:00100000:2.0:1766098640.321595:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 937
00000100:00100000:2.0:1766098640.321599:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432878208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098640.321647:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432878208:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (2378us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098640.321653:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 937
00010000:00080000:0.0:1766098640.322208:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001629b413 x1851885432878208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098647 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098640.322221:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432878208:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098640.765679:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098641.790906:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098642.813714:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098643.843340:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098644.764780:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098644.764841:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098644.764905:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098644.765021:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188736
00000100:00100000:2.0:1766098644.765103:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1979
00000100:00100000:2.0:1766098644.765111:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417188736:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098644.765205:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417188736:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 93us (448us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098644.765213:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1979
00000100:00100000:1.0:1766098644.766028:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188800
00000100:00100000:1.0:1766098644.766071:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1201
00000100:00100000:1.0:1766098644.766077:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417188800:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098644.766178:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098644.766238:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417188800:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 160us (1397us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098644.766245:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1201
00000100:00100000:3.0:1766098644.766603:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188672
00000100:00100000:3.0:1766098644.766693:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1348
00000100:00100000:3.0:1766098644.766698:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417188672:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098644.766743:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417188672:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (1839us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098644.766750:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1348
00010000:00010000:0.0:1766098644.864314:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098645.438867:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098645.438992:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0edbad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432878336:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098645.439082:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:3.0:1766098645.439136:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098645.439142:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098645.439146:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098645.439151:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3148
00000100:00080000:3.0:1766098645.439154:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098645.439160:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098645.439189:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ea384f3a x1851885432878400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098645.439270:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000019b6a62d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432878272:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.439313:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098645.439357:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878272
00000100:00100000:1.0:1766098645.439412:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1349
00000100:00100000:1.0:1766098645.439419:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432878272:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.439471:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432878272:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (158us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098645.439506:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1349
00000100:00100000:1.0:1766098645.439560:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000019b6a62d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432878272:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.439732:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dda6ad2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432878464:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.439760:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098645.439843:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea384f3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432878400:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098645.439875:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098645.439911:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878336
00000100:00100000:1.0:1766098645.439938:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878464
00000100:00100000:1.0:1766098645.439958:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1980
00000100:00100000:1.0:1766098645.439962:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b44b2785 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432878336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098645.440102:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1981
00000100:00100000:3.0:1766098645.440107:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432878464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098645.440147:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432878464:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (386us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098645.440153:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1981
00000100:00100000:3.0:1766098645.440178:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878400
00000100:00100000:1.0:1766098645.440280:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b44b2785 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432878336:12345-0@lo:400:kworker/u8:0.0 Request processed in 316us (1199us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098645.440288:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1980
00000100:00100000:1.0:1766098645.440320:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dda6ad2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432878464:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.440340:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017b40edd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432878528:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.440365:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098645.440418:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878528
00000100:00100000:2.0:1766098645.440739:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0edbad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432878336:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098645.440818:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1202
00000100:00100000:3.0:1766098645.440825:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432878400:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098645.440929:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098645.440973:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432878400:12345-0@lo:8:kworker/u8:0.0 Request processed in 148us (1098us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098645.441031:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 938
00000100:00100000:1.0:1766098645.441035:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432878528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098645.441073:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432878528:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (708us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098645.441079:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 938
00010000:00080000:1.0:1766098645.441123:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000017b40edd x1851885432878528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098652 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098645.441133:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017b40edd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432878528:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098645.441171:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098645.441175:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098645.441179:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea384f3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432878400:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098645.441309:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1202
00010000:00010000:2.0:1766098645.886106:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098646.910622:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098647.937624:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098648.958698:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098649.887113:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098649.890868:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188864
00000100:00100000:3.0:1766098649.891218:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098649.891602:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188928
00000100:00100000:0.0:1766098649.891652:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1982
00000100:00100000:0.0:1766098649.891661:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f4bdd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417188928:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098649.891759:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f4bdd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417188928:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 98us (547us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098649.891778:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1982
00000100:00100000:1.0:1766098649.892033:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1350
00000100:00100000:1.0:1766098649.892038:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417188864:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098649.892067:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417188864:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 29us (4967us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098649.892073:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1350
00000100:00100000:3.0:1766098649.892367:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098649.892404:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417188992
00000100:00100000:3.0:1766098649.892433:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1203
00000100:00100000:3.0:1766098649.892437:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417188992:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098649.892568:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098649.892600:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417188992:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 163us (235us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098649.892607:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1203
00010000:00010000:2.0:1766098649.985436:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098650.565970:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098650.565980:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098650.565986:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098650.566001:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3153
00000100:00080000:1.0:1766098650.566004:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098650.566013:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098650.566044:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d5337db0 x1851885432878720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098650.566174:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432878656:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098650.566260:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098650.566297:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098650.566302:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432878784:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098650.566320:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098650.566551:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432878720:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098650.566588:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098650.566633:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878720
00000100:00100000:1.0:1766098650.566675:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1204
00000100:00100000:1.0:1766098650.566682:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432878720:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098650.566791:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098650.566837:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432878720:12345-0@lo:8:kworker/u8:0.0 Request processed in 154us (248us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098650.566845:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1204
00000100:00080000:1.0:1766098650.566892:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098650.566896:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098650.566900:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432878720:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098650.566958:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878656
00000100:00100000:1.0:1766098650.566992:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1983
00000100:00100000:1.0:1766098650.566996:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065874e88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432878656:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.567040:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065874e88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432878656:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (782us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098650.567047:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1983
00000100:00100000:1.0:1766098650.567100:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432878848:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.567125:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098650.567141:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098650.567148:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003523eb7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432878592:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.567176:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098650.567204:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878592
00000100:00100000:1.0:1766098650.567230:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1351
00000100:00100000:1.0:1766098650.567234:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432878592:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.567266:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432878592:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (90us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098650.567272:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1351
00000100:00100000:1.0:1766098650.567298:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878848
00000100:00100000:1.0:1766098650.567321:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 939
00000100:00100000:1.0:1766098650.567323:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5337db0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432878848:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.567350:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5337db0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432878848:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (225us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098650.567355:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 939
00000100:00100000:1.0:1766098650.567375:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003523eb7f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432878592:0@lo:400:kworker/u8:0.0
00010000:00080000:1.0:1766098650.567385:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000039f6716e x1851885432878848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098657 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098650.567404:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432878848:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098650.572646:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a64620f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432878656:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.572960:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878784
00000100:00100000:1.0:1766098650.573025:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1984
00000100:00100000:1.0:1766098650.573032:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432878784:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098650.573116:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432878784:12345-0@lo:400:kworker/u8:0.0 Request processed in 85us (6796us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098650.573123:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1984
00000100:00100000:2.0:1766098650.573565:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432878784:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098651.015541:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098652.029779:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098653.054045:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098654.078660:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098655.007252:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098655.007325:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098655.007441:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098655.007443:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189056
00000100:00100000:2.0:1766098655.007588:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1352
00000100:00100000:2.0:1766098655.007595:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417189056:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098655.007668:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417189056:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 73us (343us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098655.007676:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1352
00000100:00100000:0.0:1766098655.008125:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189120
00000100:00100000:1.0:1766098655.009916:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189184
00000100:00100000:1.0:1766098655.009992:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1205
00000100:00100000:1.0:1766098655.010001:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417189184:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098655.010116:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098655.010235:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417189184:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 233us (2794us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098655.010244:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1205
00000100:00100000:0.0:1766098655.010827:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1985
00000100:00100000:0.0:1766098655.010838:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006cd3ba3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417189120:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098655.010922:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006cd3ba3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417189120:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 86us (3683us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098655.010944:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1985
00010000:00010000:2.0:1766098655.102647:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098655.678236:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098655.678242:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:0.0:1766098655.678246:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098655.678250:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098655.678261:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3159
00000100:00080000:0.0:1766098655.678265:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 6s
00000100:00080000:0.0:1766098655.678271:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098655.678280:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432878912:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678295:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000004bac994 x1851885432879040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098655.678373:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432878976:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098655.678384:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098655.678432:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057c568c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432879168:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678444:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098655.678501:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098655.678514:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098655.678529:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432879104:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678564:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098655.678566:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878912
00000100:00100000:0.0:1766098655.678616:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432878976
00000100:00100000:2.0:1766098655.678633:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1353
00000100:00100000:2.0:1766098655.678642:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432878912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678664:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879104
00000100:00100000:0.0:1766098655.678690:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1986
00000100:00100000:0.0:1766098655.678698:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e76b9049 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432878976:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098655.678715:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432878912:12345-0@lo:400:kworker/u8:0.0 Request processed in 70us (332us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098655.678737:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1353
00000100:00100000:0.0:1766098655.678779:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e76b9049 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432878976:12345-0@lo:400:kworker/u8:0.0 Request processed in 79us (338us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098655.678790:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1986
00000100:00100000:2.0:1766098655.678811:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cc1067b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432878912:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678820:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1987
00000100:00100000:0.0:1766098655.678827:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432879104:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678868:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432879104:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (303us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098655.678889:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1987
00000100:00100000:0.0:1766098655.678934:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035c0acba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432879104:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098655.678950:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432879040:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098655.678952:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432878976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098655.679006:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098655.679189:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879168
00000100:00100000:3.0:1766098655.679230:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879040
00000100:00100000:3.0:1766098655.679257:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 940
00000100:00100000:3.0:1766098655.679262:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432879168:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098655.679309:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432879168:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (836us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098655.679316:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 940
00000100:00100000:3.0:1766098655.679329:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1206
00000100:00100000:3.0:1766098655.679333:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432879040:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098655.679436:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098655.679464:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432879040:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (463us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098655.679471:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1206
00010000:00080000:3.0:1766098655.679738:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000057c568c3 x1851885432879168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098662 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098655.679765:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057c568c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432879168:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098655.680771:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098655.680779:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098655.680784:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432879040:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098656.127362:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098657.150960:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098658.173775:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098659.197788:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098660.131110:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098660.132604:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189376
00000100:00100000:1.0:1766098660.132680:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1207
00000100:00100000:1.0:1766098660.132689:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bec75682 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417189376:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098660.132808:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098660.132919:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bec75682 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417189376:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 229us (1821us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098660.132927:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1207
00000100:00100000:3.0:1766098660.133212:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098660.133383:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098660.133846:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189312
00000100:00100000:0.0:1766098660.133902:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1988
00000100:00100000:0.0:1766098660.133908:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417189312:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098660.133982:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417189312:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 74us (771us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098660.133989:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1988
00000100:00100000:0.0:1766098660.134020:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189248
00000100:00100000:0.0:1766098660.134041:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1354
00000100:00100000:0.0:1766098660.134045:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417189248:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098660.134069:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417189248:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 24us (688us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098660.134076:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1354
00010000:00010000:0.0:1766098660.225738:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098660.797788:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098660.797799:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098660.797804:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098660.797815:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3164
00000100:00080000:0.0:1766098660.797820:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 11s
00000100:00100000:3.0:1766098660.797830:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:0.0:1766098660.797845:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098660.797854:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060fbe51c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432879232:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098660.797874:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ec6e3728 x1851885432879360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098660.797934:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086644dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432879296:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098660.797944:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098660.797979:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a2ebda9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432879488:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098660.797985:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098660.798015:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098660.798015:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098660.798024:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432879424:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098660.798054:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098660.798080:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879232
00000100:00100000:0.0:1766098660.798094:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879296
00000100:00100000:3.0:1766098660.798131:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1355
00000100:00100000:0.0:1766098660.798132:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879424
00000100:00100000:3.0:1766098660.798139:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432879232:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098660.798155:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1989
00000100:00100000:0.0:1766098660.798160:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432879296:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098660.798194:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432879232:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (251us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098660.798204:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1355
00000100:00100000:0.0:1766098660.798209:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432879296:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (229us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098660.798216:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1989
00000100:00100000:0.0:1766098660.798235:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1990
00000100:00100000:0.0:1766098660.798240:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000031d74cda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432879424:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098660.798265:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060fbe51c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432879232:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098660.798271:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000031d74cda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432879424:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (216us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098660.798279:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1990
00000100:00100000:3.0:1766098660.798306:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879488
00000100:00100000:0.0:1766098660.798327:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432879424:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098660.798336:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 941
00000100:00100000:0.0:1766098660.798342:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086644dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432879296:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098660.798343:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432879488:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098660.798379:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432879360:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766098660.798391:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432879488:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (375us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098660.798398:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 941
00000100:00100000:0.0:1766098660.798412:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766098660.798425:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002a2ebda9 x1851885432879488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098667 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766098660.798434:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a2ebda9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432879488:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766098660.798497:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879360
00000100:00100000:3.0:1766098660.798519:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1208
00000100:00100000:3.0:1766098660.798522:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432879360:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766098660.798623:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098660.798665:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432879360:12345-0@lo:8:kworker/u8:2.0 Request processed in 141us (254us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098660.798671:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1208
00000100:00080000:0.0:1766098660.801376:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098660.801389:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098660.801399:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432879360:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766098661.245969:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098662.269863:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098663.294531:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098664.319143:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098665.255801:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098665.255849:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098665.255895:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098665.256580:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189440
00000100:00100000:0.0:1766098665.256640:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1356
00000100:00100000:0.0:1766098665.256647:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417189440:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098665.256722:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417189440:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 75us (928us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098665.256729:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1356
00000100:00100000:2.0:1766098665.258176:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189504
00000100:00100000:2.0:1766098665.258282:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1991
00000100:00100000:2.0:1766098665.258289:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417189504:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098665.258383:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417189504:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 93us (2534us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098665.258391:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1991
00000100:00100000:1.0:1766098665.261953:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189568
00000100:00100000:1.0:1766098665.262038:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1209
00000100:00100000:1.0:1766098665.262048:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417189568:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098665.262153:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098665.262232:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417189568:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 184us (6337us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098665.262239:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1209
00010000:00010000:2.0:1766098665.341874:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098665.919236:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098665.919245:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098665.919249:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098665.919260:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3169
00000100:00080000:3.0:1766098665.919264:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 16s
00000100:00080000:3.0:1766098665.919271:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098665.919295:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a75c5a33 x1851885432879680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098665.919352:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a67325cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432879808:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098665.919432:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098665.919467:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098665.919471:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432879552:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098665.919685:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432879744:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098665.919771:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a75c5a33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432879680:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098665.921173:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098665.921226:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098665.921238:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc97a667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432879616:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098665.921276:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098665.921332:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879744
00000100:00100000:1.0:1766098665.921382:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879616
00000100:00100000:1.0:1766098665.921406:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1992
00000100:00100000:1.0:1766098665.921413:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432879744:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098665.921470:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432879744:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (303us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098665.921492:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1992
00000100:00100000:1.0:1766098665.921512:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1993
00000100:00100000:1.0:1766098665.921516:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432879616:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098665.921564:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432879616:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (288us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098665.921571:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1993
00000100:00100000:1.0:1766098665.921624:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc97a667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432879616:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098665.921635:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432879744:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098665.922539:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879808
00000100:00100000:0.0:1766098665.922584:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 942
00000100:00100000:0.0:1766098665.922589:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432879808:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098665.922663:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432879808:12345-0@lo:400:kworker/u8:0.0 Request processed in 73us (3232us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098665.922669:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 942
00000100:00100000:2.0:1766098665.924033:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098665.924240:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879680
00000100:00100000:3.0:1766098665.924883:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766098665.924934:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a67325cf x1851885432879808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098672 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098665.924976:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a67325cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432879808:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098665.925025:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879552
00000100:00100000:3.0:1766098665.925057:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1357
00000100:00100000:3.0:1766098665.925062:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432879552:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098665.925111:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432879552:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (230us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098665.925117:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1357
00000100:00100000:3.0:1766098665.925154:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432879552:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098665.925313:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1210
00000100:00100000:0.0:1766098665.925319:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432879680:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098665.925413:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098665.925450:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432879680:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (1423us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098665.925455:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1210
00000100:00080000:3.0:1766098665.925583:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098665.925588:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098665.925592:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a75c5a33 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432879680:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766098666.369947:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098667.389689:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098668.417717:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766098669.440914:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098670.367873:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098670.367930:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098670.367955:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189696
00000100:00100000:2.0:1766098670.368017:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1994
00000100:00100000:2.0:1766098670.368024:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a75c5a33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417189696:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098670.368097:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a75c5a33 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417189696:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 73us (233us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098670.368104:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1994
00000100:00100000:0.0:1766098670.368194:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189632
00000100:00100000:0.0:1766098670.368233:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1358
00000100:00100000:0.0:1766098670.368254:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417189632:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098670.368296:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417189632:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 42us (366us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098670.368303:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1358
00000100:00100000:3.0:1766098670.369017:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098670.369063:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189760
00000100:00100000:1.0:1766098670.369922:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1211
00000100:00100000:1.0:1766098670.369929:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417189760:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098670.370028:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098670.370103:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417189760:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 173us (1088us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098670.370110:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1211
00010000:00010000:2.0:1766098670.461711:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098671.037723:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098671.037730:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098671.037733:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098671.037741:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3174
00000100:00080000:0.0:1766098671.037745:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098671.037750:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098671.037771:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000142c5414 x1851885432880000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098671.037914:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432879872:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098671.037979:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098671.038002:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098671.038007:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092653fe0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432880128:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098671.038041:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098671.038067:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000142c5414 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880000:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098671.038088:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098671.038119:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880128
00000100:00100000:3.0:1766098671.038151:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880000
00000100:00100000:3.0:1766098671.038184:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 943
00000100:00100000:3.0:1766098671.038189:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e914bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432880128:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098671.038370:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879872
00000100:00100000:0.0:1766098671.038394:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1359
00000100:00100000:0.0:1766098671.038398:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432879872:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098671.038445:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432879872:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (467us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098671.038452:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1359
00000100:00100000:0.0:1766098671.038497:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1212
00000100:00100000:0.0:1766098671.038501:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432880000:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098671.038596:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098671.038639:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432880000:12345-0@lo:8:kworker/u8:0.0 Request processed in 136us (550us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098671.038646:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1212
00000100:00100000:0.0:1766098671.038682:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432879872:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098671.038731:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098671.038734:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098671.038738:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000142c5414 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880000:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098671.038835:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e914bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432880128:12345-0@lo:400:kworker/u8:0.0 Request processed in 644us (793us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098671.038842:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 943
00000100:00100000:2.0:1766098671.038932:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432879936:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098671.038962:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098671.038978:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098671.038983:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432880064:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098671.039001:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098671.039028:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432879936
00000100:00100000:2.0:1766098671.039051:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880064
00000100:00100000:2.0:1766098671.039067:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1995
00000100:00100000:2.0:1766098671.039071:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432879936:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098671.039109:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432879936:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (146us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098671.039115:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1995
00000100:00100000:2.0:1766098671.039126:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1996
00000100:00100000:2.0:1766098671.039130:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1f26126 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432880064:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098671.039151:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1f26126 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432880064:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (149us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098671.039156:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1996
00000100:00100000:2.0:1766098671.039183:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432880064:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098671.039191:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432879936:0@lo:400:kworker/u8:0.0
00010000:00080000:0.0:1766098671.039259:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000092653fe0 x1851885432880128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098678 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098671.039268:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092653fe0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432880128:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098671.487232:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098672.509713:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098673.533763:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098674.559281:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098675.494455:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098675.494531:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098675.494573:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098675.494634:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189952
00000100:00100000:2.0:1766098675.494672:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1213
00000100:00100000:2.0:1766098675.494679:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417189952:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098675.494776:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098675.494830:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417189952:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 151us (257us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098675.494837:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1213
00000100:00100000:0.0:1766098675.495088:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189888
00000100:00100000:0.0:1766098675.495129:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1997
00000100:00100000:0.0:1766098675.495134:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008834c3ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417189888:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098675.495171:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008834c3ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417189888:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 37us (641us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098675.495177:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1997
00000100:00100000:3.0:1766098675.496442:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417189824
00000100:00100000:3.0:1766098675.496526:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1360
00000100:00100000:3.0:1766098675.496531:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417189824:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098675.496588:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417189824:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 56us (2140us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098675.496594:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1360
00010000:00010000:0.0:1766098675.581868:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098676.157782:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098676.157801:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098676.157804:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098676.157811:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3179
00000100:00080000:2.0:1766098676.157814:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098676.157819:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098676.157840:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000addcb14d x1851885432880320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098676.157893:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432880256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098676.157956:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098676.157995:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098676.158001:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432880384:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098676.158022:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098676.158055:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880256
00000100:00100000:2.0:1766098676.158095:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880384
00000100:00100000:2.0:1766098676.158117:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1998
00000100:00100000:2.0:1766098676.158123:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432880256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098676.158185:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035c0acba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432880256:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (229us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098676.158193:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1998
00000100:00100000:2.0:1766098676.158210:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1999
00000100:00100000:2.0:1766098676.158213:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432880384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098676.158239:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002cc1067b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432880384:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (217us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098676.158245:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1999
00000100:00100000:2.0:1766098676.158284:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004bac994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432880384:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766098676.158306:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432880256:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158391:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432880448:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158419:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098676.158444:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098676.158448:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432880192:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158468:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098676.158504:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880320:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158522:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098676.158548:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880192
00000100:00100000:0.0:1766098676.158568:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1361
00000100:00100000:0.0:1766098676.158588:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432880192:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158615:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432880192:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (147us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098676.158620:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1361
00000100:00100000:0.0:1766098676.158647:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880448
00000100:00100000:0.0:1766098676.158663:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880320
00000100:00100000:0.0:1766098676.158673:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 944
00000100:00100000:0.0:1766098676.158676:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653fe0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432880448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158696:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653fe0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432880448:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (278us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098676.158701:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 944
00000100:00100000:0.0:1766098676.158709:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1214
00000100:00100000:0.0:1766098676.158711:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432880320:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766098676.158798:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098676.158823:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432880320:12345-0@lo:8:kworker/u8:2.0 Request processed in 110us (300us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098676.158829:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1214
00000100:00080000:0.0:1766098676.158856:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098676.158859:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098676.158863:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880320:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098676.158884:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432880192:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766098676.158892:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000afdf3ced x1851885432880448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098683 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766098676.158901:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afdf3ced pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432880448:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766098676.605758:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098677.629751:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098678.653975:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098679.677710:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098680.604818:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098680.604942:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190144
00000100:00100000:2.0:1766098680.604995:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1215
00000100:00100000:2.0:1766098680.605002:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417190144:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098680.605104:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098680.605178:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417190144:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 175us (369us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098680.605186:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1215
00000100:00100000:3.0:1766098680.607504:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098680.607622:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190016
00000100:00100000:0.0:1766098680.607678:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1362
00000100:00100000:0.0:1766098680.607685:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417190016:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098680.607758:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417190016:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (289us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098680.607765:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1362
00000100:00100000:1.0:1766098680.608229:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098680.608299:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190080
00000100:00100000:0.0:1766098680.608335:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2000
00000100:00100000:0.0:1766098680.608340:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417190080:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098680.608384:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417190080:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 43us (156us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098680.608391:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2000
00010000:00010000:2.0:1766098680.701751:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098681.278374:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098681.278381:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098681.278384:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098681.278391:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3184
00000100:00080000:0.0:1766098681.278393:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098681.278398:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098681.278418:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000069da214f x1851885432880640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098681.278461:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039d784a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432880576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278553:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098681.278575:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a2ebda9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432880768:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278581:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098681.278598:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432880704:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098681.278605:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098681.278620:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098681.278623:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098681.278627:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432880512:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278640:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880640:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766098681.278651:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098681.278662:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098681.278691:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880576
00000100:00100000:1.0:1766098681.278691:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880768
00000100:00100000:0.0:1766098681.278726:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880704
00000100:00100000:1.0:1766098681.278727:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880640
00000100:00100000:0.0:1766098681.278744:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2001
00000100:00100000:1.0:1766098681.278744:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 945
00000100:00100000:0.0:1766098681.278749:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060fbe51c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432880576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766098681.278749:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432880768:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278790:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060fbe51c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432880576:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (237us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098681.278790:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432880768:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (184us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098681.278798:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2001
00000100:00100000:1.0:1766098681.278798:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 945
00000100:00100000:0.0:1766098681.278812:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2002
00000100:00100000:1.0:1766098681.278812:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1216
00000100:00100000:1.0:1766098681.278815:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432880640:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278816:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057c568c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432880704:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278836:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057c568c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432880704:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (216us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098681.278841:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2002
00000100:00100000:0.0:1766098681.278879:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432880704:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278887:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039d784a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432880576:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766098681.278909:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098681.278927:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880512
00000100:00100000:1.0:1766098681.278937:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432880640:12345-0@lo:8:kworker/u8:2.0 Request processed in 121us (275us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098681.278943:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1216
00000100:00100000:0.0:1766098681.278948:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1363
00000100:00100000:0.0:1766098681.278951:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432880512:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766098681.278966:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098681.278970:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098681.278973:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766098681.278975:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432880512:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (324us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098681.278980:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1363
00000100:00100000:1.0:1766098681.278991:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432880512:0@lo:400:kworker/u8:2.0
00010000:00080000:1.0:1766098681.278998:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002a2ebda9 x1851885432880768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098688 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766098681.279007:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a2ebda9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432880768:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766098681.725736:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098682.749815:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098683.773943:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098684.798095:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098685.752046:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098685.752088:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098685.752133:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190208
00000100:00100000:3.0:1766098685.752160:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098685.752704:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190272
00000100:00100000:1.0:1766098685.752754:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2003
00000100:00100000:1.0:1766098685.752760:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000663baa05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417190272:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098685.752811:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1364
00000100:00100000:0.0:1766098685.752818:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417190208:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098685.752837:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000663baa05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417190272:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 75us (748us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098685.752846:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2003
00000100:00100000:0.0:1766098685.752853:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417190208:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 32us (812us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098685.752861:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1364
00000100:00100000:0.0:1766098685.752914:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190336
00000100:00100000:0.0:1766098685.752938:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1217
00000100:00100000:0.0:1766098685.752943:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417190336:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098685.753036:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098685.755518:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417190336:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 2555us (3341us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098685.755534:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1217
00010000:00010000:2.0:1766098685.822018:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098686.401469:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:0.0:1766098686.402696:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098686.402703:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098686.402706:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098686.402713:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3189
00000100:00080000:0.0:1766098686.402716:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098686.402721:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098686.402738:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b7ebad2d x1851885432880960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098686.402940:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432881088:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403001:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098686.403026:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432880832:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403050:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098686.403083:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7ebad2d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880960:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403103:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098686.403131:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881088
00000100:00100000:1.0:1766098686.403163:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880960
00000100:00100000:1.0:1766098686.403178:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 946
00000100:00100000:1.0:1766098686.403183:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432881088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403222:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432881088:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (222us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098686.403229:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 946
00000100:00100000:1.0:1766098686.403255:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1218
00000100:00100000:1.0:1766098686.403258:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432880960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098686.403358:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098686.403384:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432880960:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (281us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098686.403390:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1218
00000100:00080000:1.0:1766098686.403424:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098686.403427:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098686.403430:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7ebad2d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432880960:0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766098686.403457:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000aea833b9 x1851885432881088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098693 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098686.403466:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432881088:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403533:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432880896:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403559:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098686.403578:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098686.403583:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed297b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432881024:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403602:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098686.403626:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880896
00000100:00100000:1.0:1766098686.403649:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881024
00000100:00100000:1.0:1766098686.403664:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2004
00000100:00100000:1.0:1766098686.403668:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a46965fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432880896:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403698:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a46965fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432880896:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (139us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098686.403704:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2004
00000100:00100000:1.0:1766098686.403716:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2005
00000100:00100000:1.0:1766098686.403719:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444deb29 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432881024:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403741:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444deb29 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432881024:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (140us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098686.403747:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2005
00000100:00100000:1.0:1766098686.403773:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed297b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432881024:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403780:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432880896:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403814:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432880832
00000100:00100000:1.0:1766098686.403831:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1365
00000100:00100000:1.0:1766098686.403835:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432880832:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098686.403860:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432880832:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (811us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098686.403874:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1365
00000100:00100000:1.0:1766098686.403903:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432880832:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098686.845723:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098687.869765:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098688.893775:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098689.918239:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098690.846795:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098690.846863:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098690.847011:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190400
00000100:00100000:0.0:1766098690.847081:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1366
00000100:00100000:0.0:1766098690.847109:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417190400:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098690.847198:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417190400:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 89us (414us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098690.847206:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1366
00000100:00100000:1.0:1766098690.847349:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190464
00000100:00100000:1.0:1766098690.847393:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2006
00000100:00100000:1.0:1766098690.847398:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417190464:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098690.847434:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004bac994 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417190464:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 36us (572us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098690.847441:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2006
00000100:00100000:0.0:1766098690.850386:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098690.851767:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190528
00000100:00100000:2.0:1766098690.851842:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1219
00000100:00100000:2.0:1766098690.851850:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417190528:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098690.851962:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098690.852059:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417190528:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 209us (1685us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098690.852084:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1219
00010000:00010000:0.0:1766098690.941853:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098691.517865:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098691.517873:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098691.517877:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:0.0:1766098691.517882:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002810083f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432881152:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766098691.517888:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3194
00000100:00080000:1.0:1766098691.517902:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766098691.517909:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098691.517934:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000a6b62b0 x1851885432881280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098691.517984:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098691.517994:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432881216:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098691.518026:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098691.518035:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432881408:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518040:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098691.518076:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098691.518079:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098691.518088:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432881344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518120:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098691.518127:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881216
00000100:00100000:1.0:1766098691.518153:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432881280:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518180:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098691.518184:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881344
00000100:00100000:0.0:1766098691.518221:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2007
00000100:00100000:0.0:1766098691.518229:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432881216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518252:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2008
00000100:00100000:1.0:1766098691.518258:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc68c7d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432881344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098691.518297:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a3ed9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432881216:12345-0@lo:400:kworker/u8:3.0 Request processed in 67us (259us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098691.518298:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881152
00000100:00100000:1.0:1766098691.518304:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc68c7d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432881344:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (182us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098691.518310:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2007
00000100:00100000:1.0:1766098691.518319:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2008
00000100:00100000:2.0:1766098691.518342:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1367
00000100:00100000:2.0:1766098691.518348:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432881152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518365:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881408
00000100:00100000:0.0:1766098691.518397:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047fe747c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432881344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518399:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881280
00000100:00100000:2.0:1766098691.518401:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432881152:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (418us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098691.518411:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1367
00000100:00100000:0.0:1766098691.518419:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba96c12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432881216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518431:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 947
00000100:00100000:1.0:1766098691.518437:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079a59718 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432881408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098691.518518:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1220
00000100:00100000:1.0:1766098691.518522:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079a59718 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432881408:12345-0@lo:400:kworker/u8:3.0 Request processed in 84us (445us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098691.518527:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432881280:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518533:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 947
00010000:00080000:1.0:1766098691.518597:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000affb1b2b x1851885432881408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098698 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766098691.518606:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432881408:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098691.518615:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002810083f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432881152:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766098691.518630:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098691.518685:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432881280:12345-0@lo:8:kworker/u8:3.0 Request processed in 160us (505us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098691.518699:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1220
00000100:00080000:1.0:1766098691.518792:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098691.518796:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098691.518799:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432881280:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766098691.966753:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098692.990586:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098694.013842:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098695.038005:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098695.965217:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098695.970219:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098695.970873:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098695.970911:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190656
00000100:00100000:0.0:1766098695.971000:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2009
00000100:00100000:0.0:1766098695.971008:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417190656:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098695.971101:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417190656:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 93us (5896us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098695.971109:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2009
00000100:00100000:3.0:1766098695.972306:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190720
00000100:00100000:3.0:1766098695.972361:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1221
00000100:00100000:3.0:1766098695.972368:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d431ef27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417190720:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766098695.972508:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190592
00000100:00100000:2.0:1766098695.972559:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1368
00000100:00100000:2.0:1766098695.972565:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417190592:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098695.972647:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417190592:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 81us (2439us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098695.972654:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1368
00010000:02020000:3.0:1766098695.973914:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098695.973979:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d431ef27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417190720:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1610us (3109us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098695.973990:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1221
00010000:00010000:2.0:1766098696.065511:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098696.638570:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098696.638580:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098696.638584:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098696.638595:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3199
00000100:00080000:0.0:1766098696.638599:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098696.638606:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098696.638634:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a478d4ad x1851885432881600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098696.638689:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432881536:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098696.638769:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098696.638810:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098696.638814:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432881664:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098696.638844:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098696.638880:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881536
00000100:00100000:0.0:1766098696.638921:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881664
00000100:00100000:0.0:1766098696.638938:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2010
00000100:00100000:0.0:1766098696.638944:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068c3be10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432881536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098696.638980:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2011
00000100:00100000:0.0:1766098696.638984:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045cb1471 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432881664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098696.639031:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045cb1471 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432881664:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (186us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098696.639039:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2011
00000100:00100000:0.0:1766098696.639092:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432881664:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098696.639141:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068c3be10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432881536:12345-0@lo:400:kworker/u8:0.0 Request processed in 197us (374us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098696.639147:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2010
00000100:00100000:0.0:1766098696.639167:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432881536:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098696.639350:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432881728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098696.639382:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098696.639415:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098696.639431:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8700567 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432881472:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098696.639462:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098696.639510:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881472
00000100:00100000:3.0:1766098696.639534:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1369
00000100:00100000:3.0:1766098696.639537:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432881472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098696.639571:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432881472:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (108us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098696.639577:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1369
00000100:00100000:3.0:1766098696.639609:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8700567 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432881472:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098696.639680:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881728
00000100:00100000:2.0:1766098696.639719:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948
00000100:00100000:2.0:1766098696.639722:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432881728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098696.639775:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432881728:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (392us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098696.639782:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948
00010000:00080000:2.0:1766098696.639806:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000010ae602d x1851885432881728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098703 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098696.639837:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432881728:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098696.639979:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a478d4ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432881600:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098696.640013:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098696.640108:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881600
00000100:00100000:2.0:1766098696.640136:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1222
00000100:00100000:2.0:1766098696.640139:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432881600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098696.640234:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098696.640268:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432881600:12345-0@lo:8:kworker/u8:0.0 Request processed in 127us (255us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098696.640273:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1222
00000100:00080000:1.0:1766098696.640641:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098696.640646:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098696.640650:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a478d4ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432881600:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766098697.086621:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098698.109908:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098699.133692:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098700.158380:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098701.085163:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098701.087827:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190912
00000100:00100000:3.0:1766098701.087904:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1223
00000100:00100000:3.0:1766098701.087913:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417190912:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766098701.088618:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098701.088669:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098701.088999:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190784
00000100:00100000:0.0:1766098701.089056:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1370
00000100:00100000:0.0:1766098701.089063:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417190784:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098701.089149:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417190784:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 86us (534us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098701.089158:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1370
00000100:00100000:1.0:1766098701.089645:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190848
00000100:00100000:1.0:1766098701.089791:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2012
00000100:00100000:1.0:1766098701.089799:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417190848:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098701.089848:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afdf3ced pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417190848:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 49us (1178us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098701.089856:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2012
00010000:02020000:3.0:1766098701.090846:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098701.090904:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417190912:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 2991us (5753us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098701.090915:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1223
00010000:00010000:2.0:1766098701.185209:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098701.757676:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432881792:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098701.757680:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098701.757695:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098701.757699:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098701.757705:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3205
00000100:00080000:2.0:1766098701.757708:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098701.757715:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098701.757736:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e6e5460e x1851885432881920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098701.757751:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098701.757775:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098701.757804:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432881856:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098701.757814:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881792
00000100:00100000:0.0:1766098701.757863:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1371
00000100:00100000:0.0:1766098701.757871:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432881792:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098701.757894:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098701.757944:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432881920:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098701.757956:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ce9aa68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432881984:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098701.757965:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432881792:12345-0@lo:400:kworker/u8:0.0 Request processed in 88us (210us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098701.757976:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1371
00000100:00100000:2.0:1766098701.757997:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098701.758026:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098701.758041:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432881792:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098701.758063:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098701.758075:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009db79f91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432882048:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098701.758104:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098701.758143:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881856
00000100:00100000:0.0:1766098701.758166:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881984
00000100:00100000:0.0:1766098701.758181:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013
00000100:00100000:0.0:1766098701.758185:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432881856:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098701.758197:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2014
00000100:00100000:2.0:1766098701.758201:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432881984:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098701.758221:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a67325cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432881856:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (337us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098701.758227:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013
00000100:00100000:0.0:1766098701.758260:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432881920
00000100:00100000:0.0:1766098701.758275:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882048
00000100:00100000:0.0:1766098701.758292:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 949
00000100:00100000:0.0:1766098701.758296:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432882048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098701.758323:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432882048:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (221us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098701.758330:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 949
00000100:00100000:0.0:1766098701.758343:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1224
00000100:00100000:0.0:1766098701.758347:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432881920:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098701.758360:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008050eca6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432881984:12345-0@lo:400:kworker/u8:0.0 Request processed in 156us (339us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098701.758369:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2014
00010000:00080000:1.0:1766098701.758398:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009db79f91 x1851885432882048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098708 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098701.758408:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009db79f91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432882048:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766098701.758451:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098701.758506:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432881920:12345-0@lo:8:kworker/u8:0.0 Request processed in 158us (512us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098701.758517:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1224
00000100:00080000:2.0:1766098701.758634:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098701.758638:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098701.758641:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432881920:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098701.758760:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ce9aa68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432881984:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098701.758775:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432881856:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098702.205989:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098703.229779:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098704.253955:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098705.277727:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098706.206121:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098706.206470:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098706.208457:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417190976
00000100:00100000:0.0:1766098706.211809:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1372
00000100:00100000:0.0:1766098706.211827:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417190976:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098706.211904:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191040
00000100:00100000:0.0:1766098706.211964:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417190976:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 140us (5857us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098706.211973:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1372
00000100:00100000:2.0:1766098706.212039:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2015
00000100:00100000:2.0:1766098706.212046:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e914bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417191040:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098706.212088:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e914bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417191040:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 41us (5619us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098706.212096:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2015
00000100:00100000:3.0:1766098706.212260:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098706.212440:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191104
00000100:00100000:0.0:1766098706.212504:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1225
00000100:00100000:0.0:1766098706.212511:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417191104:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098706.215316:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098706.215460:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417191104:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 2947us (3199us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098706.215493:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1225
00010000:00010000:0.0:1766098706.301927:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098706.877686:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432882112:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098706.877691:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1766098706.877700:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432882176:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098706.877700:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098706.877704:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098706.877709:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3210
00000100:00080000:2.0:1766098706.877712:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098706.877719:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098706.877744:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000addcb14d x1851885432882240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098706.877786:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432882368:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098706.877881:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098706.877941:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882112
00000100:00100000:1.0:1766098706.877998:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1373
00000100:00100000:1.0:1766098706.878007:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432882112:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098706.878064:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432882112:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (185us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098706.878074:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1373
00000100:00100000:1.0:1766098706.878129:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432882112:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098706.878165:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098706.878208:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882368
00000100:00100000:1.0:1766098706.878259:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 950
00000100:00100000:1.0:1766098706.878264:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432882368:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098706.878279:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432882304:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098706.878307:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432882368:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (143us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098706.878324:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098706.878332:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 950
00000100:00100000:2.0:1766098706.878373:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432882240:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098706.878391:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882304
00000100:00100000:2.0:1766098706.878401:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098706.878439:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2016
00010000:00080000:2.0:1766098706.878440:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cbff49d7 x1851885432882368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098713 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098706.878447:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432882304:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098706.878461:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432882368:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098706.878499:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098706.878506:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882240
00000100:00100000:1.0:1766098706.878526:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432882304:12345-0@lo:400:kworker/u8:0.0 Request processed in 79us (202us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098706.878532:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1226
00000100:00100000:1.0:1766098706.878538:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2016
00000100:00100000:0.0:1766098706.878539:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882176
00000100:00100000:2.0:1766098706.878539:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432882240:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098706.878637:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098706.878671:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432882240:12345-0@lo:8:kworker/u8:0.0 Request processed in 133us (271us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098706.878678:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1226
00000100:00080000:2.0:1766098706.878701:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098706.878705:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098706.878708:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432882240:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098706.878728:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432882304:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098706.878954:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2017
00000100:00100000:0.0:1766098706.878960:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432882176:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098706.878999:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432882176:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (525us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098706.879017:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2017
00000100:00100000:1.0:1766098706.879356:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432882176:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098707.326005:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098708.350611:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098709.377163:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098710.397847:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098711.328567:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098711.329997:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098711.330057:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098711.330123:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191296
00000100:00100000:3.0:1766098711.330150:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191232
00000100:00100000:0.0:1766098711.330190:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1227
00000100:00100000:3.0:1766098711.330201:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2018
00000100:00100000:0.0:1766098711.330203:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417191296:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:3.0:1766098711.330211:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417191232:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098711.330297:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417191232:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 89us (307us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098711.330305:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2018
00010000:02020000:0.0:1766098711.330308:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098711.330362:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417191296:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 163us (1808us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098711.330370:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1227
00000100:00100000:1.0:1766098711.330548:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191168
00000100:00100000:1.0:1766098711.330595:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1374
00000100:00100000:1.0:1766098711.330599:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417191168:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098711.330654:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417191168:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 46us (589us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098711.330662:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1374
00010000:00010000:0.0:1766098711.422646:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098711.997729:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432882432:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098711.997760:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1766098711.997776:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432882496:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098711.997788:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098711.997793:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098711.997799:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3215
00000100:00080000:2.0:1766098711.997803:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098711.997811:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098711.997855:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e6e5460e x1851885432882560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098711.997874:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098711.997919:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098711.997928:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432882624:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098711.997935:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432882560:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098711.998009:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432882688:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098711.998092:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098711.998136:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882496
00000100:00100000:0.0:1766098711.998188:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882624
00000100:00100000:0.0:1766098711.998215:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2019
00000100:00100000:2.0:1766098711.998217:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098711.998225:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000983dda28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432882496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098711.998246:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098711.998263:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882560
00000100:00100000:0.0:1766098711.998284:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000983dda28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432882496:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (413us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098711.998294:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1228
00000100:00100000:0.0:1766098711.998295:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2019
00000100:00100000:2.0:1766098711.998302:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432882560:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098711.998320:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2020
00000100:00100000:0.0:1766098711.998366:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432882496:0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766098711.998420:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098711.998462:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432882560:12345-0@lo:8:kworker/u8:0.0 Request processed in 161us (249us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098711.998471:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1228
00000100:00100000:2.0:1766098711.998525:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882432
00000100:00100000:2.0:1766098711.998560:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1375
00000100:00100000:2.0:1766098711.998565:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432882432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098711.998610:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432882432:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (364us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098711.998617:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1375
00000100:00080000:2.0:1766098711.998667:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098711.998671:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098711.998675:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432882560:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098711.998716:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbff49d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432882432:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098711.999041:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000da5bbb9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432882624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098711.999098:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000da5bbb9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432882624:12345-0@lo:400:kworker/u8:0.0 Request processed in 772us (1006us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098711.999116:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2020
00000100:00100000:2.0:1766098711.999418:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2232645 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432882624:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098711.999580:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098711.999864:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882688
00000100:00100000:2.0:1766098712.000590:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 951
00000100:00100000:2.0:1766098712.000599:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432882688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098712.000678:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432882688:12345-0@lo:400:kworker/u8:0.0 Request processed in 79us (1099us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098712.000686:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 951
00010000:00080000:3.0:1766098712.001171:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005114f03e x1851885432882688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098718 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098712.001187:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432882688:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098712.448845:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098713.486799:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098714.493795:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098715.517697:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098716.444535:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098716.444647:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191360
00000100:00100000:2.0:1766098716.445294:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1376
00000100:00100000:2.0:1766098716.445303:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417191360:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098716.445393:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417191360:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 89us (870us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098716.445402:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1376
00000100:00100000:3.0:1766098716.453899:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098716.454009:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098716.454131:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191424
00000100:00100000:1.0:1766098716.454198:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2021
00000100:00100000:1.0:1766098716.454206:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095954fe9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417191424:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098716.454310:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095954fe9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417191424:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 104us (423us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098716.454318:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2021
00000100:00100000:2.0:1766098716.454859:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191488
00000100:00100000:2.0:1766098716.454895:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1229
00000100:00100000:2.0:1766098716.454900:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c786030 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417191488:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098716.455035:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098716.455073:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c786030 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417191488:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 173us (1064us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098716.455080:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1229
00010000:00010000:2.0:1766098716.547190:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098717.117974:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432882752:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098717.118076:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098717.118138:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882752
00000100:00100000:2.0:1766098717.118200:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1377
00000100:00100000:2.0:1766098717.118208:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432882752:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766098717.119371:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098717.119378:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098717.119382:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098717.119389:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3220
00000100:00080000:1.0:1766098717.119393:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766098717.119399:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098717.119428:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009b57ab98 x1851885432882880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098717.120290:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432882752:12345-0@lo:400:kworker/u8:3.0 Request processed in 2080us (2214us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098717.120292:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432882816:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098717.120304:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1377
00000100:00100000:1.0:1766098717.120351:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098717.120380:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098717.120384:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432882944:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098717.120384:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432883008:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098717.120407:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098717.120410:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098717.120436:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432882880:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098717.120453:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882816
00000100:00100000:1.0:1766098717.120464:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098717.120509:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882944
00000100:00100000:1.0:1766098717.120511:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2022
00000100:00100000:1.0:1766098717.120516:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432882816:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098717.120532:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2023
00000100:00100000:2.0:1766098717.120535:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e10322b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432882944:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098717.120672:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883008
00000100:00100000:0.0:1766098717.120701:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432882880
00000100:00100000:0.0:1766098717.120717:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 952
00000100:00100000:0.0:1766098717.120721:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2232645 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432883008:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098717.120761:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2232645 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432883008:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (351us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098717.120767:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 952
00000100:00100000:0.0:1766098717.120781:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1230
00000100:00100000:0.0:1766098717.120784:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432882880:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098717.120879:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098717.120911:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432882880:12345-0@lo:8:kworker/u8:3.0 Request processed in 125us (446us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098717.120917:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1230
00000100:00100000:3.0:1766098717.121779:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432882752:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766098717.121938:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b6bb6a94 x1851885432883008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098724 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098717.121950:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432883008:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098717.122616:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e10322b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432882944:12345-0@lo:400:kworker/u8:3.0 Request processed in 2077us (2206us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098717.122626:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2023
00000100:00100000:0.0:1766098717.122929:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432882944:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098717.122946:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432882816:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766098717.122993:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098717.122997:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098717.123001:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432882880:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098717.123299:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432882816:12345-0@lo:400:kworker/u8:3.0 Request processed in 2747us (2913us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098717.123308:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2022
00010000:00010000:0.0:1766098717.565862:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098718.590797:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098719.613848:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098720.637760:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098721.565354:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098721.565763:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098721.565803:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098721.566396:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191552
00000100:00100000:2.0:1766098721.566427:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191616
00000100:00100000:2.0:1766098721.566536:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2024
00000100:00100000:2.0:1766098721.566545:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417191616:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098721.566638:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417191616:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 93us (1295us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098721.566646:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2024
00000100:00100000:0.0:1766098721.568990:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191680
00000100:00100000:0.0:1766098721.569069:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1231
00000100:00100000:0.0:1766098721.569077:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c1ee83e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417191680:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098721.569178:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098721.569285:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c1ee83e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417191680:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 207us (3482us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098721.569293:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1231
00000100:00100000:3.0:1766098721.570303:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1378
00000100:00100000:3.0:1766098721.570309:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417191552:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098721.570361:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417191552:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 51us (4599us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098721.570367:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1378
00010000:00010000:2.0:1766098721.661729:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098722.238410:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766098722.238411:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1766098722.238433:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432883072:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098722.238441:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098722.238447:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098722.238453:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3225
00000100:00080000:2.0:1766098722.238456:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098722.238464:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098722.238529:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a2360adf x1851885432883200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098722.238545:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098722.238590:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432883328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098722.238621:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098722.238624:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883072
00000100:00100000:0.0:1766098722.238932:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432883136:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098722.238970:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098722.238993:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098722.238997:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432883264:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098722.239019:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098722.239046:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432883200:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098722.239069:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098722.239100:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883328
00000100:00100000:0.0:1766098722.239134:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883200
00000100:00100000:0.0:1766098722.239157:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 953
00000100:00100000:0.0:1766098722.239163:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432883328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098722.239222:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432883328:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (600us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098722.239242:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 953
00000100:00100000:0.0:1766098722.239265:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1232
00000100:00100000:0.0:1766098722.239268:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8700567 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432883200:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098722.239366:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098722.239377:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1379
00000100:00100000:2.0:1766098722.239384:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432883072:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098722.239404:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8700567 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432883200:12345-0@lo:8:kworker/u8:0.0 Request processed in 133us (333us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098722.239411:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1232
00000100:00100000:2.0:1766098722.239431:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432883072:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (887us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098722.239440:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1379
00000100:00100000:1.0:1766098722.239442:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883136
00000100:00080000:0.0:1766098722.239463:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098722.239468:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098722.239501:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883264
00000100:00100000:2.0:1766098722.239507:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025
00000100:00100000:2.0:1766098722.239512:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a478d4ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432883136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098722.239525:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2360adf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432883200:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098722.239529:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2026
00000100:00100000:1.0:1766098722.239536:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432883264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098722.239550:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a478d4ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432883136:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (579us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098722.239556:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025
00000100:00100000:1.0:1766098722.239570:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432883264:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (552us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098722.239576:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2026
00000100:00100000:2.0:1766098722.239591:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432883264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098722.239602:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432883136:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766098722.239769:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004fe5d403 x1851885432883328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098729 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098722.239780:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432883328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098722.239790:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432883072:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098722.685765:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098723.710046:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098724.734603:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098725.758393:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098726.685143:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098726.685260:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191872
00000100:00100000:0.0:1766098726.685706:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1233
00000100:00100000:0.0:1766098726.685713:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417191872:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098726.685809:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098726.687240:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417191872:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1523us (2100us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098726.687253:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1233
00000100:00100000:1.0:1766098726.694471:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098726.694572:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098726.694714:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191808
00000100:00100000:3.0:1766098726.694770:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2027
00000100:00100000:3.0:1766098726.694777:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000458dfff5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417191808:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098726.694934:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191744
00000100:00100000:2.0:1766098726.694975:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1380
00000100:00100000:2.0:1766098726.694979:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417191744:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098726.695684:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000458dfff5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417191808:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 905us (1110us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098726.695694:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2027
00000100:00100000:2.0:1766098726.695864:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417191744:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 883us (1400us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098726.695871:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1380
00010000:00010000:0.0:1766098726.782253:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098727.359710:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098727.359718:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098727.359721:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098727.359729:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3230
00000100:00080000:3.0:1766098727.359745:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098727.359750:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098727.359772:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000437b852e x1851885432883520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098727.359849:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432883584:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098727.359923:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098727.359959:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098727.359965:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432883456:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098727.359985:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098727.360006:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432883392:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098727.360032:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098727.360046:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098727.360050:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432883648:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098727.360071:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098727.360142:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883584
00000100:00100000:3.0:1766098727.360183:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883456
00000100:00100000:3.0:1766098727.360231:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2028
00000100:00100000:3.0:1766098727.360237:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5354685 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432883584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098727.360290:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5354685 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432883584:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (367us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098727.360297:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2028
00000100:00100000:3.0:1766098727.360313:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2029
00000100:00100000:3.0:1766098727.360316:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c5bf9cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432883456:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098727.360338:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c5bf9cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432883456:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (354us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098727.360343:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2029
00000100:00100000:3.0:1766098727.360380:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432883456:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098727.360389:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432883584:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098727.360471:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883392
00000100:00100000:2.0:1766098727.360519:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1381
00000100:00100000:2.0:1766098727.360523:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432883392:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098727.360566:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432883392:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (534us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098727.360572:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1381
00000100:00100000:2.0:1766098727.360606:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432883392:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098727.360753:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432883520:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098727.360784:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098727.360812:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883648
00000100:00100000:0.0:1766098727.360834:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883520
00000100:00100000:0.0:1766098727.360847:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 954
00000100:00100000:0.0:1766098727.360851:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432883648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098727.360887:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432883648:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (815us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098727.360892:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 954
00000100:00100000:0.0:1766098727.360904:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1234
00000100:00100000:0.0:1766098727.360906:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432883520:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098727.360993:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098727.361021:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432883520:12345-0@lo:8:kworker/u8:3.0 Request processed in 113us (237us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098727.361027:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1234
00000100:00080000:0.0:1766098727.361061:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098727.361065:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098727.361067:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432883520:0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766098727.361220:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008bdf5e63 x1851885432883648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098734 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766098727.361229:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432883648:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098727.810397:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098728.829663:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098729.855395:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098730.878032:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098731.808639:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098731.809167:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417191936
00000100:00100000:3.0:1766098731.809246:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1382
00000100:00100000:3.0:1766098731.809255:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417191936:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098731.809354:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417191936:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 100us (729us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098731.809362:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1382
00000100:00100000:1.0:1766098731.809395:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098731.809465:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192000
00000100:00100000:3.0:1766098731.809545:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2030
00000100:00100000:3.0:1766098731.809552:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d75985ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417192000:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098731.809624:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d75985ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417192000:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (238us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098731.809631:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2030
00000100:00100000:1.0:1766098731.813565:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098731.813762:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192064
00000100:00100000:0.0:1766098731.814463:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1235
00000100:00100000:0.0:1766098731.814473:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417192064:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098731.814604:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098731.814686:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417192064:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 213us (1128us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098731.814694:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1235
00010000:00010000:2.0:1766098731.902124:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098732.481558:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098732.481565:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098732.481569:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766098732.481575:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:1.0:1766098732.481578:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3235
00000100:00080000:1.0:1766098732.481582:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766098732.481587:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098732.481595:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432883712:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098732.481608:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000092653886 x1851885432883840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098732.481667:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432883840:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098732.481679:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098732.481715:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4d173d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432883968:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098732.481716:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098732.481762:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098732.481764:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009bbfcf2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432883776:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098732.481799:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098732.481810:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883840
00000100:00100000:1.0:1766098732.481819:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098732.481826:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432883904:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098732.481847:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098732.481859:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883968
00000100:00100000:1.0:1766098732.481891:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 955
00000100:00100000:1.0:1766098732.481899:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432883968:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098732.481902:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1236
00000100:00100000:2.0:1766098732.481909:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432883840:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098732.481948:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432883968:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (185us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098732.481955:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 955
00000100:00100000:1.0:1766098732.481978:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883712
00010000:02020000:2.0:1766098732.481999:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098732.482003:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1383
00000100:00100000:1.0:1766098732.482007:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432883712:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098732.482056:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432883840:12345-0@lo:8:kworker/u8:3.0 Request processed in 145us (341us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098732.482059:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432883712:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (387us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098732.482069:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1236
00000100:00100000:1.0:1766098732.482072:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1383
00000100:00100000:2.0:1766098732.482098:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883776
00000100:00100000:2.0:1766098732.482127:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432883904
00000100:00080000:1.0:1766098732.482151:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:2.0:1766098732.482161:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2031
00000100:00100000:2.0:1766098732.482167:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432883776:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766098732.482168:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098732.482175:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432883840:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098732.482215:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2032
00000100:00100000:2.0:1766098732.482215:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432883776:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (412us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098732.482224:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2031
00000100:00100000:1.0:1766098732.482233:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a846069e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432883904:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098732.482259:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009bbfcf2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432883776:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098732.482270:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a846069e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432883904:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (421us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098732.482277:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2032
00000100:00100000:2.0:1766098732.482283:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432883904:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766098732.482310:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a4d173d7 x1851885432883968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098739 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766098732.482318:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4d173d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432883968:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098732.482327:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432883712:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098732.926284:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098733.949717:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098734.973776:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098735.999367:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098736.928559:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098736.928604:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098736.928679:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192192
00000100:00100000:3.0:1766098736.928731:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2033
00000100:00100000:3.0:1766098736.928738:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ecb9adb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417192192:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098736.928820:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ecb9adb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417192192:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 82us (216us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098736.928828:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2033
00000100:00100000:3.0:1766098736.929199:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098736.929291:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192128
00000100:00100000:2.0:1766098736.929330:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1384
00000100:00100000:2.0:1766098736.929334:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417192128:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098736.929369:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417192128:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (171us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098736.929376:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1384
00000100:00100000:0.0:1766098736.929605:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192256
00000100:00100000:0.0:1766098736.929638:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1237
00000100:00100000:0.0:1766098736.929642:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417192256:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098736.929738:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098736.929781:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417192256:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 138us (1230us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098736.929788:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1237
00010000:00010000:2.0:1766098737.022309:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098737.598363:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098737.598371:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098737.598374:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098737.598382:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3240
00000100:00080000:2.0:1766098737.598386:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098737.598393:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098737.598412:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007baf6919 x1851885432884160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098737.598455:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432884288:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098737.598549:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098737.598580:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098737.598585:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2a90eed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432884032:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098737.598611:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098737.598630:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007baf6919 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432884160:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098737.598655:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884288
00000100:00100000:1.0:1766098737.598658:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098737.598690:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884160
00000100:00100000:2.0:1766098737.598720:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 956
00000100:00100000:2.0:1766098737.598727:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432884288:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098737.598732:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432884096:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098737.598766:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098737.598772:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432884288:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (237us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098737.598783:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098737.598787:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432884224:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098737.598795:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 956
00000100:00100000:0.0:1766098737.598804:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098737.598811:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1238
00000100:00100000:2.0:1766098737.598814:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432884160:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098737.598831:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884096
00000100:00100000:0.0:1766098737.598854:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884224
00000100:00100000:0.0:1766098737.598867:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2034
00000100:00100000:0.0:1766098737.598871:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432884096:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098737.598899:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432884096:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (133us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098737.598904:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2034
00010000:02020000:2.0:1766098737.598906:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098737.598916:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2035
00000100:00100000:0.0:1766098737.598919:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079a59718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432884224:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098737.598932:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432884160:12345-0@lo:8:kworker/u8:0.0 Request processed in 116us (275us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098737.598937:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079a59718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432884224:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (133us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098737.598938:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1238
00000100:00100000:0.0:1766098737.598942:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2035
00010000:00080000:2.0:1766098737.598969:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000df6e7141 x1851885432884288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098744 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098737.598980:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432884288:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098737.599124:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098737.599128:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098737.599132:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007baf6919 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432884160:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098737.599162:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432884224:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098737.599171:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432884096:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098737.599270:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884032
00000100:00100000:3.0:1766098737.600107:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1385
00000100:00100000:3.0:1766098737.600125:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432884032:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098737.600210:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432884032:12345-0@lo:400:kworker/u8:0.0 Request processed in 94us (1598us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098737.600217:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1385
00000100:00100000:1.0:1766098737.601267:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2a90eed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432884032:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098738.045668:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098739.070345:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098740.093788:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098741.119473:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098742.049911:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098742.049961:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098742.049995:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192320
00000100:00100000:0.0:1766098742.050147:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192384
00000100:00100000:0.0:1766098742.050185:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2036
00000100:00100000:0.0:1766098742.050192:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417192384:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098742.050272:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417192384:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 80us (311us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098742.050281:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2036
00000100:00100000:3.0:1766098742.050345:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098742.050376:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192448
00000100:00100000:3.0:1766098742.051044:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1239
00000100:00100000:3.0:1766098742.051049:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417192448:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098742.051538:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098742.051641:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1386
00000100:00100000:2.0:1766098742.051648:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417192320:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098742.053600:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417192320:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1943us (3690us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098742.053618:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1386
00000100:00100000:3.0:1766098742.053632:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417192448:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 2579us (3285us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098742.053643:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1239
00010000:00010000:0.0:1766098742.142206:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098742.718072:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432884352:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098742.718154:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766098742.718237:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098742.718255:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098742.718258:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098742.718262:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3246
00000100:00080000:1.0:1766098742.718265:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766098742.718270:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098742.718292:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a9f903ce x1851885432884480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098742.718398:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884352
00000100:00100000:0.0:1766098742.718439:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1387
00000100:00100000:0.0:1766098742.718445:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432884352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098742.718627:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092cf9228 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432884608:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098742.718652:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098742.718676:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432884544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098742.718693:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098742.718704:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098742.718707:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432884416:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098742.718717:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098742.718733:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884608
00000100:00100000:3.0:1766098742.718754:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 957
00000100:00100000:3.0:1766098742.718757:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432884608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098742.718766:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9f903ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432884480:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098742.718783:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432884608:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (130us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098742.718788:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 957
00000100:00100000:1.0:1766098742.718805:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766098742.718817:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000092cf9228 x1851885432884608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098749 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098742.718826:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092cf9228 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432884608:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098742.718847:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884480
00000100:00100000:3.0:1766098742.718859:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1240
00000100:00100000:3.0:1766098742.718862:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432884480:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098742.718870:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884544
00000100:00100000:1.0:1766098742.718900:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884416
00000100:00100000:1.0:1766098742.718920:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037
00000100:00100000:1.0:1766098742.718924:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000437b852e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432884544:12345-0@lo:400:kworker/u8:0.0
00010000:02020000:3.0:1766098742.718941:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098742.718962:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000437b852e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432884544:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (266us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098742.718970:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037
00000100:00100000:3.0:1766098742.718981:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432884480:12345-0@lo:8:kworker/u8:0.0 Request processed in 116us (178us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098742.718987:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1240
00000100:00100000:1.0:1766098742.718989:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2038
00000100:00100000:1.0:1766098742.718996:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b57ab98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432884416:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098742.719009:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098742.719012:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098742.719014:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9f903ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432884480:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098742.719035:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b57ab98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432884416:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (315us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098742.719042:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2038
00000100:00100000:3.0:1766098742.719046:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432884416:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098742.719062:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432884544:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098742.719324:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432884352:12345-0@lo:400:kworker/u8:0.0 Request processed in 877us (1169us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098742.719330:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1387
00000100:00100000:2.0:1766098742.719418:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432884352:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098743.165682:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098744.190880:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098745.213870:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098746.239533:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098747.170247:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098747.170297:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098747.170334:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098747.170424:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192576
00000100:00100000:3.0:1766098747.170508:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2039
00000100:00100000:3.0:1766098747.170513:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417192576:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.170584:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417192576:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 70us (345us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098747.170590:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2039
00000100:00100000:1.0:1766098747.170775:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192640
00000100:00100000:1.0:1766098747.170806:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1241
00000100:00100000:1.0:1766098747.170811:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000362df469 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417192640:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098747.170905:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098747.170957:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000362df469 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417192640:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 145us (623us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098747.170963:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1241
00000100:00100000:2.0:1766098747.171983:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192512
00000100:00100000:2.0:1766098747.172031:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1388
00000100:00100000:2.0:1766098747.172036:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417192512:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098747.172104:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417192512:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 67us (1807us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098747.172111:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1388
00010000:00010000:2.0:1766098747.262036:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098747.838720:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098747.838728:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098747.838733:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098747.838740:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3251
00000100:00080000:2.0:1766098747.838744:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00100000:0.0:1766098747.838745:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076159a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432884736:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766098747.838751:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098747.838775:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d2a90eed x1851885432884800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098747.838810:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098747.838824:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432884864:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098747.838858:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098747.838888:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2a90eed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432884800:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098747.838914:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098747.839048:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432884928:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.839079:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098747.839106:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098747.839112:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432884672:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.839136:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098747.839168:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884736
00000100:00100000:3.0:1766098747.839202:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884864
00000100:00100000:3.0:1766098747.839221:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2040
00000100:00100000:3.0:1766098747.839226:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed297b67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432884736:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.839274:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed297b67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432884736:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (465us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098747.839295:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2040
00000100:00100000:3.0:1766098747.839315:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2041
00000100:00100000:3.0:1766098747.839318:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007baf6919 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432884864:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.839359:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007baf6919 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432884864:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (501us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098747.839364:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2041
00000100:00100000:0.0:1766098747.839368:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076159a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432884736:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.839389:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884800
00000100:00100000:0.0:1766098747.839419:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884672
00000100:00100000:0.0:1766098747.839449:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1389
00000100:00100000:0.0:1766098747.839454:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432884672:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098747.840361:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884928
00000100:00100000:0.0:1766098747.840615:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432884672:12345-0@lo:400:kworker/u8:3.0 Request processed in 1158us (1476us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098747.840623:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1389
00000100:00100000:2.0:1766098747.840833:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 958
00000100:00100000:2.0:1766098747.840838:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432884928:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098747.840890:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432884928:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (1810us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098747.840909:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 958
00000100:00100000:2.0:1766098747.840924:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1242
00000100:00100000:2.0:1766098747.840928:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432884800:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766098747.841031:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098747.841086:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432884800:12345-0@lo:8:kworker/u8:3.0 Request processed in 157us (2172us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098747.841093:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1242
00000100:00100000:2.0:1766098747.841138:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432884672:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766098747.841149:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000da267b78 x1851885432884928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098754 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766098747.841159:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432884928:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766098747.841232:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098747.841238:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098747.841241:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2a90eed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432884800:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098747.841272:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432884864:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098748.288541:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098749.309791:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098750.333663:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098751.358276:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098752.288769:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098752.288817:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098752.288851:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192704
00000100:00100000:3.0:1766098752.288855:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098752.288894:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1390
00000100:00100000:2.0:1766098752.288900:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417192704:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098752.288955:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192832
00000100:00100000:2.0:1766098752.288983:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417192704:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 82us (222us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098752.288988:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1243
00000100:00100000:2.0:1766098752.288991:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1390
00000100:00100000:0.0:1766098752.288992:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417192832:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098752.289109:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098752.289137:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417192832:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 144us (282us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098752.289144:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1243
00000100:00100000:1.0:1766098752.289724:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192768
00000100:00100000:1.0:1766098752.289771:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2042
00000100:00100000:1.0:1766098752.289775:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417192768:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098752.289831:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417192768:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 55us (1014us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098752.289838:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2042
00010000:00010000:2.0:1766098752.383300:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098752.957692:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098752.958244:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432884992:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098752.958316:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098752.958352:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432884992
00000100:00080000:3.0:1766098752.958500:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098752.958505:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098752.958508:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098752.958513:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3256
00000100:00080000:3.0:1766098752.958515:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098752.958520:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098752.958539:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000039f6716e x1851885432885120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098752.958580:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ea27af9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432885184:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.958608:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098752.958624:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098752.958629:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432885056:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.958646:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098752.958666:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432885120:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098752.958701:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098752.958728:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885184
00000100:00100000:3.0:1766098752.958751:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885056
00000100:00100000:3.0:1766098752.958766:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2043
00000100:00100000:3.0:1766098752.958772:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432885184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.958814:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432885184:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (206us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098752.958821:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2043
00000100:00100000:3.0:1766098752.958837:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2044
00000100:00100000:3.0:1766098752.958840:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432885056:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.958861:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432885056:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (214us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098752.958867:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2044
00000100:00100000:3.0:1766098752.958884:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885120
00000100:00100000:3.0:1766098752.958899:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1244
00000100:00100000:3.0:1766098752.958902:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432885120:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766098752.959014:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098752.959043:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432885120:12345-0@lo:8:kworker/u8:3.0 Request processed in 140us (341us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098752.959049:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1244
00000100:00080000:3.0:1766098752.959081:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098752.959084:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098752.959088:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432885120:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098752.959113:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432885056:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.959121:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ea27af9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432885184:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.959146:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af55eb76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432885248:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.959168:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098752.959190:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885248
00000100:00100000:3.0:1766098752.959229:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 959
00000100:00100000:3.0:1766098752.959233:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432885248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098752.959260:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432885248:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (93us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098752.959266:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 959
00010000:00080000:3.0:1766098752.959286:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000af55eb76 x1851885432885248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098759 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098752.959295:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af55eb76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432885248:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098752.959608:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1391
00000100:00100000:1.0:1766098752.959615:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432884992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098752.959663:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432884992:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (1349us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098752.959670:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1391
00000100:00100000:3.0:1766098752.963378:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432884992:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098753.409049:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098754.429675:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098755.453681:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098756.477772:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098757.406016:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098757.406077:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098757.406145:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192896
00000100:00100000:0.0:1766098757.406173:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417192960
00000100:00100000:2.0:1766098757.406223:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1392
00000100:00100000:0.0:1766098757.406229:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2045
00000100:00100000:2.0:1766098757.406234:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417192896:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098757.406243:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417192960:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098757.406328:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417192896:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 98us (324us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098757.406336:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1392
00000100:00100000:0.0:1766098757.406647:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417192960:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 407us (569us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098757.406654:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2045
00000100:00100000:2.0:1766098757.406966:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098757.407034:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193024
00000100:00100000:1.0:1766098757.407696:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1245
00000100:00100000:1.0:1766098757.407702:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193024:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098757.407812:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098757.407868:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193024:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 164us (904us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098757.407877:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1245
00010000:00010000:0.0:1766098757.506220:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098758.080026:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098758.080036:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098758.080040:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098758.080049:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3261
00000100:00080000:0.0:1766098758.080053:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098758.080061:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098758.080090:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bb97adba x1851885432885440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098758.080150:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7ebad2d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432885376:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080230:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098758.080272:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098758.080277:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432885504:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080301:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098758.080341:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885376
00000100:00100000:0.0:1766098758.080389:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885504
00000100:00100000:0.0:1766098758.080413:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2046
00000100:00100000:0.0:1766098758.080419:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432885376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080495:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2360adf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432885376:12345-0@lo:400:kworker/u8:3.0 Request processed in 75us (266us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098758.080504:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2046
00000100:00100000:0.0:1766098758.080548:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2047
00000100:00100000:0.0:1766098758.080552:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbff49d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432885504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080580:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbff49d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432885504:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (279us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098758.080586:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2047
00000100:00100000:0.0:1766098758.080625:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6c1f26a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432885504:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080637:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7ebad2d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432885376:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080698:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432885440:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080726:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098758.080756:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432885568:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080777:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098758.080792:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098758.080796:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432885312:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.080822:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098758.080936:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885440
00000100:00100000:0.0:1766098758.080957:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885568
00000100:00100000:0.0:1766098758.080999:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960
00000100:00100000:0.0:1766098758.081002:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432885568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098758.081036:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432885568:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (259us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098758.081042:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960
00000100:00100000:0.0:1766098758.081053:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1246
00000100:00100000:0.0:1766098758.081056:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021cadba5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432885440:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098758.081151:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098758.081194:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021cadba5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432885440:12345-0@lo:8:kworker/u8:3.0 Request processed in 136us (468us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098758.081201:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1246
00000100:00080000:0.0:1766098758.081350:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098758.081354:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098758.081358:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432885440:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766098758.081384:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005114f03e x1851885432885568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098765 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766098758.081394:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432885568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098758.081701:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885312
00000100:00100000:1.0:1766098758.081752:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1393
00000100:00100000:1.0:1766098758.081756:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432885312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098758.081801:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432885312:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (979us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098758.081806:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1393
00000100:00100000:1.0:1766098758.081836:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432885312:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098758.529100:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098759.549776:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098760.573721:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098761.597739:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098762.528006:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098762.528076:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098762.528141:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193152
00000100:00100000:0.0:1766098762.528220:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2048
00000100:00100000:0.0:1766098762.528228:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417193152:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098762.528326:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417193152:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 97us (332us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098762.528334:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2048
00000100:00100000:2.0:1766098762.528445:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193088
00000100:00100000:2.0:1766098762.528514:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1394
00000100:00100000:2.0:1766098762.528519:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417193088:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098762.528549:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417193088:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 30us (474us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098762.528556:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1394
00000100:00100000:0.0:1766098762.531697:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098762.533044:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193216
00000100:00100000:3.0:1766098762.533124:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1247
00000100:00100000:3.0:1766098762.533131:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417193216:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098762.533238:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098762.533345:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417193216:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 213us (1657us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098762.533655:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1247
00010000:00010000:2.0:1766098762.621782:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098763.197704:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098763.197725:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098763.197730:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:0.0:1766098763.197737:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432885696:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098763.197741:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3266
00000100:00080000:3.0:1766098763.197750:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098763.197759:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098763.197785:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000039f6716e x1851885432885760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098763.197844:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098763.197869:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432885824:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098763.197922:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432885760:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098763.197950:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098763.198035:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432885888:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098763.198080:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098763.198129:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885696
00000100:00100000:0.0:1766098763.198173:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885824
00000100:00100000:0.0:1766098763.198190:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2049
00000100:00100000:0.0:1766098763.198196:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432885696:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098763.198258:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432885696:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (417us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098763.198266:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2049
00000100:00100000:0.0:1766098763.198284:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2050
00000100:00100000:0.0:1766098763.198287:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432885824:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098763.198307:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432885824:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (361us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098763.198311:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2050
00000100:00100000:0.0:1766098763.198329:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885760
00000100:00100000:0.0:1766098763.198349:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1248
00000100:00100000:0.0:1766098763.198354:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432885760:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098763.198366:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432885824:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098763.198452:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432885632:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766098763.198454:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098763.198531:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098763.198545:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432885760:12345-0@lo:8:kworker/u8:0.0 Request processed in 189us (463us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098763.198554:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1248
00000100:00100000:0.0:1766098763.198589:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885632
00000100:00100000:3.0:1766098763.198620:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5337db0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432885696:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098763.198633:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1395
00000100:00100000:0.0:1766098763.198639:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432885632:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098763.198688:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098763.198694:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098763.198698:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6716e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432885760:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098763.201246:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432885632:12345-0@lo:400:kworker/u8:0.0 Request processed in 2595us (2708us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098763.201266:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1395
00000100:00100000:3.0:1766098763.202658:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432885632:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098763.202671:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098763.203722:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885888
00000100:00100000:2.0:1766098763.203801:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 961
00000100:00100000:2.0:1766098763.203809:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000677df168 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432885888:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098763.203918:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000677df168 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432885888:12345-0@lo:400:kworker/u8:0.0 Request processed in 108us (1257us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098763.203935:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 961
00010000:00080000:0.0:1766098763.204089:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003a108df5 x1851885432885888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098770 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098763.204106:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432885888:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098763.645762:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098764.669876:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098765.693681:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098766.725205:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098767.666436:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098767.668410:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193344
00000100:00100000:3.0:1766098767.668514:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2051
00000100:00100000:3.0:1766098767.668592:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417193344:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098767.668674:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417193344:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 83us (2248us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098767.668682:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2051
00000100:00100000:1.0:1766098767.669152:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098767.669321:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098767.669388:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193280
00000100:00100000:0.0:1766098767.669426:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1396
00000100:00100000:0.0:1766098767.669431:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417193280:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098767.669491:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417193280:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 43us (324us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098767.669499:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1396
00000100:00100000:2.0:1766098767.670678:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193408
00000100:00100000:2.0:1766098767.670718:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1249
00000100:00100000:2.0:1766098767.670723:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193408:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098767.670820:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098767.671230:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193408:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 505us (1908us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098767.671239:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1249
00010000:00010000:2.0:1766098767.742571:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098768.319195:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098768.319202:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098768.319205:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098768.319212:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3271
00000100:00080000:0.0:1766098768.319215:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098768.319220:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098768.319239:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000fe6e47c6 x1851885432886080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098768.319363:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432886016:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319427:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098768.319466:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098768.319471:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432886144:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319502:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098768.319525:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432886080:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319544:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098768.319580:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886016
00000100:00100000:3.0:1766098768.319612:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886144
00000100:00100000:3.0:1766098768.319626:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2052
00000100:00100000:3.0:1766098768.319631:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432886016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319669:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432886016:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (243us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098768.319676:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2052
00000100:00100000:3.0:1766098768.319690:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2053
00000100:00100000:3.0:1766098768.319693:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432886144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319709:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432886144:12345-0@lo:400:kworker/u8:3.0 Request processed in 16us (208us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098768.319714:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2053
00000100:00100000:3.0:1766098768.319742:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432886144:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319749:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432886016:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319772:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432885952:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319794:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098768.319806:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098768.319823:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432886208:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319840:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098768.319858:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432885952
00000100:00100000:3.0:1766098768.319879:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1397
00000100:00100000:3.0:1766098768.319882:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432885952:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098768.319918:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432885952:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (124us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098768.319924:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1397
00000100:00100000:3.0:1766098768.319946:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432885952:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098768.320061:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886080
00000100:00100000:0.0:1766098768.320106:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886208
00000100:00100000:0.0:1766098768.320145:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 962
00000100:00100000:0.0:1766098768.320149:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432886208:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098768.320178:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432886208:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (338us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098768.320183:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 962
00000100:00100000:0.0:1766098768.320193:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1250
00000100:00100000:0.0:1766098768.320196:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432886080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098768.320326:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098768.320352:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432886080:12345-0@lo:8:kworker/u8:3.0 Request processed in 156us (808us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098768.320358:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1250
00000100:00080000:0.0:1766098768.320378:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098768.320381:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098768.320384:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432886080:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766098768.320540:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cc7d0c6a x1851885432886208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098775 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098768.320550:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432886208:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098768.766275:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098769.789847:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098770.813700:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098771.837820:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098772.766844:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098772.766911:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098772.768290:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193600
00000100:00100000:3.0:1766098772.768372:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1251
00000100:00100000:3.0:1766098772.768380:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193600:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098772.768525:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098772.768614:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193600:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 234us (1780us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098772.768623:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1251
00000100:00100000:1.0:1766098772.770441:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193472
00000100:00100000:1.0:1766098772.770533:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1398
00000100:00100000:1.0:1766098772.770541:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417193472:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098772.770623:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417193472:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 83us (3712us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098772.770631:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1398
00000100:00100000:0.0:1766098772.770866:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098772.770942:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193536
00000100:00100000:2.0:1766098772.770986:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2054
00000100:00100000:2.0:1766098772.770991:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417193536:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098772.771043:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9b941c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417193536:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 51us (191us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098772.771050:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2054
00010000:00010000:0.0:1766098772.864699:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098773.437902:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098773.437910:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098773.437914:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098773.437923:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3276
00000100:00080000:3.0:1766098773.437927:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098773.437951:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098773.437977:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000fe6e47c6 x1851885432886400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098773.438035:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432886336:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098773.438112:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098773.438149:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098773.438153:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432886464:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098773.438171:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098773.438196:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432886400:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098773.438332:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432886528:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098773.438524:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432886272:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.438816:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098773.438862:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098773.438888:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886528
00000100:00100000:1.0:1766098773.438950:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 963
00000100:00100000:1.0:1766098773.438957:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432886528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439031:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432886528:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (216us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098773.439041:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 963
00010000:00080000:1.0:1766098773.439092:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e445b90d x1851885432886528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098780 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098773.439115:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432886528:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439175:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886272
00000100:00100000:1.0:1766098773.439206:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1399
00000100:00100000:1.0:1766098773.439209:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432886272:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439253:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432886272:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (403us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098773.439259:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1399
00000100:00100000:1.0:1766098773.439290:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886336
00000100:00100000:1.0:1766098773.439312:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886464
00000100:00100000:1.0:1766098773.439326:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2055
00000100:00100000:1.0:1766098773.439329:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432886336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439359:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432886336:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (1248us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098773.439365:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2055
00000100:00100000:1.0:1766098773.439374:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2056
00000100:00100000:1.0:1766098773.439377:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5337db0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432886464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439407:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5337db0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432886464:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (1236us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098773.439413:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2056
00000100:00100000:1.0:1766098773.439439:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432886272:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439583:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432886464:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098773.439593:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002684e04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432886336:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098773.439899:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098773.440528:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886400
00000100:00100000:1.0:1766098773.440566:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1252
00000100:00100000:1.0:1766098773.440570:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432886400:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098773.440763:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098773.440923:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432886400:12345-0@lo:8:kworker/u8:0.0 Request processed in 352us (1025us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098773.440931:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1252
00000100:00080000:3.0:1766098773.440997:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098773.441002:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098773.441006:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432886400:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098773.886234:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098774.912062:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098775.933756:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098776.957771:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098777.885124:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098777.885173:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098777.885214:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098777.885423:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193792
00000100:00100000:0.0:1766098777.885469:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1253
00000100:00100000:0.0:1766098777.885512:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193792:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098777.885608:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098777.885667:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417193792:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 190us (550us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098777.885675:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1253
00000100:00100000:3.0:1766098777.885812:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193728
00000100:00100000:0.0:1766098777.887619:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2057
00000100:00100000:0.0:1766098777.887632:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417193728:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098777.887737:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417193728:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 105us (2563us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098777.887745:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2057
00000100:00100000:1.0:1766098777.888995:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193664
00000100:00100000:1.0:1766098777.889043:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1400
00000100:00100000:1.0:1766098777.889049:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417193664:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098777.890675:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417193664:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1619us (5455us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098777.890690:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1400
00010000:00010000:2.0:1766098777.981761:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098778.557839:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098778.557846:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098778.557850:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098778.557871:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3281
00000100:00080000:2.0:1766098778.557875:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098778.557881:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098778.557905:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004fe5d403 x1851885432886720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098778.557966:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432886848:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098778.557967:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432886784:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098778.558064:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432886592:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098778.558444:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098778.558449:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098778.558501:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098778.558509:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432886656:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098778.558532:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886848
00000100:00100000:2.0:1766098778.558547:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098778.558586:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 964
00000100:00100000:2.0:1766098778.558586:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432886720:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098778.558604:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432886848:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098778.558617:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098778.558664:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886784
00000100:00100000:1.0:1766098778.558670:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432886848:12345-0@lo:400:kworker/u8:0.0 Request processed in 65us (221us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098778.558679:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 964
00000100:00100000:2.0:1766098778.558699:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886656
00000100:00100000:1.0:1766098778.558704:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886720
00000100:00100000:2.0:1766098778.558722:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2058
00000100:00100000:2.0:1766098778.558728:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a90eed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432886784:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098778.558734:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1254
00000100:00100000:1.0:1766098778.558740:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432886720:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098778.558763:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2059
00000100:00100000:2.0:1766098778.558767:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a90eed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432886784:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (324us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098778.558772:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076159a69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432886656:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098778.558778:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2058
00000100:00100000:2.0:1766098778.558825:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432886784:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098778.558830:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076159a69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432886656:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (283us total) trans 0 rc 0/0
00010000:02020000:1.0:1766098778.558837:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098778.558841:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2059
00000100:00100000:2.0:1766098778.558848:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432886656:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098778.558889:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432886720:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (272us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098778.558897:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1254
00000100:00080000:2.0:1766098778.558927:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098778.558931:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098778.558935:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432886720:0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766098778.558938:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000061093366 x1851885432886848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098785 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098778.558954:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432886848:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098778.559246:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098778.559327:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886592
00000100:00100000:1.0:1766098778.559353:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1401
00000100:00100000:1.0:1766098778.559356:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432886592:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098778.559394:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432886592:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (148us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098778.559400:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1401
00000100:00100000:0.0:1766098778.559460:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432886592:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098779.007915:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098780.030160:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098781.053703:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098782.077687:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098783.005244:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098783.005283:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098783.005355:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193856
00000100:00100000:2.0:1766098783.005358:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193920
00000100:00100000:2.0:1766098783.005409:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2060
00000100:00100000:2.0:1766098783.005415:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c786030 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417193920:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098783.005457:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1402
00000100:00100000:3.0:1766098783.005464:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417193856:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098783.005518:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c786030 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417193920:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 101us (234us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098783.005530:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2060
00000100:00100000:3.0:1766098783.005530:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417193856:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 62us (288us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098783.005541:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1402
00000100:00100000:0.0:1766098783.005938:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098783.007194:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417193984
00000100:00100000:3.0:1766098783.007238:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1255
00000100:00100000:3.0:1766098783.007243:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417193984:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098783.007335:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098783.007399:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417193984:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 154us (1461us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098783.007406:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1255
00010000:00010000:2.0:1766098783.102287:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098783.677971:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098783.678655:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432886912:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098783.678737:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098783.679613:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886912
00000100:00080000:3.0:1766098783.679746:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098783.679751:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098783.679754:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098783.679761:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3287
00000100:00080000:3.0:1766098783.679764:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766098783.679769:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098783.679799:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000070167dff x1851885432887040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098783.680148:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1403
00000100:00100000:2.0:1766098783.680155:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432886912:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098783.680223:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432886912:12345-0@lo:400:kworker/u8:3.0 Request processed in 68us (1488us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098783.680230:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1403
00000100:00100000:3.0:1766098783.680643:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432887040:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098783.680682:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098783.680717:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432886976:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098783.680741:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098783.680760:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098783.680765:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432887104:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098783.680797:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098783.681071:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432887168:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098783.681103:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098783.681273:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432886976
00000100:00100000:1.0:1766098783.681293:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887104
00000100:00100000:3.0:1766098783.681319:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061
00000100:00100000:3.0:1766098783.681325:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001cc58289 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432886976:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098783.681399:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001cc58289 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432886976:12345-0@lo:400:kworker/u8:3.0 Request processed in 73us (657us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098783.681406:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061
00000100:00100000:3.0:1766098783.681428:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2062
00000100:00100000:3.0:1766098783.681432:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018d302a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432887104:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098783.681470:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018d302a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432887104:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (674us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098783.681503:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2062
00000100:00100000:3.0:1766098783.681544:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432887104:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098783.682366:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432886912:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098783.682610:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432886976:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098783.682960:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887040
00000100:00100000:0.0:1766098783.683000:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887168
00000100:00100000:0.0:1766098783.683028:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 965
00000100:00100000:0.0:1766098783.683033:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432887168:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098783.683102:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432887168:12345-0@lo:400:kworker/u8:3.0 Request processed in 68us (2000us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098783.683110:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 965
00000100:00100000:0.0:1766098783.683127:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1256
00000100:00100000:0.0:1766098783.683131:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432887040:12345-0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766098783.683224:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000319ec734 x1851885432887168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098790 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00010000:02020000:0.0:1766098783.683233:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098783.683239:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432887168:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098783.683272:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432887040:12345-0@lo:8:kworker/u8:3.0 Request processed in 139us (2589us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098783.683279:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1256
00000100:00080000:2.0:1766098783.683298:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098783.683303:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098783.683307:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432887040:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766098784.125910:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098785.149799:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098786.176014:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098787.199219:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098788.124414:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098788.124457:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098788.124524:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098788.124623:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194048
00000100:00100000:1.0:1766098788.124691:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1404
00000100:00100000:1.0:1766098788.124697:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194048:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098788.124792:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194048:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 95us (268us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098788.124800:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1404
00000100:00100000:3.0:1766098788.124910:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194176
00000100:00100000:3.0:1766098788.124946:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1257
00000100:00100000:3.0:1766098788.124951:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002569b802 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417194176:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098788.125058:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098788.125089:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002569b802 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417194176:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 137us (632us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098788.125095:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1257
00000100:00100000:0.0:1766098788.125166:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194112
00000100:00100000:0.0:1766098788.125206:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2063
00000100:00100000:0.0:1766098788.125211:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebfd82a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194112:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098788.125242:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebfd82a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194112:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 30us (836us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098788.125248:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2063
00010000:00010000:0.0:1766098788.221663:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098788.798505:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098788.798512:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098788.798516:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098788.798524:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3292
00000100:00080000:0.0:1766098788.798527:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098788.798534:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098788.798534:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098788.798554:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d35037c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432887296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098788.798558:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008c1ee83e x1851885432887360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098788.798620:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098788.798654:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad887d7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432887424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098788.798655:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c1ee83e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432887360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098788.798694:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098788.798701:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098788.798769:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887360
00000100:00100000:0.0:1766098788.798801:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1258
00000100:00100000:0.0:1766098788.798806:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432887360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098788.798896:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098788.798935:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432887360:12345-0@lo:8:kworker/u8:3.0 Request processed in 128us (235us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098788.798943:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1258
00000100:00100000:1.0:1766098788.798952:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432887232:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098788.798993:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766098788.798999:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098788.799004:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098788.799010:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c1ee83e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432887360:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098788.799018:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098788.799027:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432887488:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098788.799046:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887296
00000100:00100000:1.0:1766098788.799066:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098788.799075:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887424
00000100:00100000:0.0:1766098788.799101:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2064
00000100:00100000:1.0:1766098788.799101:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887488
00000100:00100000:0.0:1766098788.799107:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a7d48c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432887296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098788.799134:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 966
00000100:00100000:1.0:1766098788.799140:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432887488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098788.799153:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a7d48c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432887296:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (534us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098788.799160:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2064
00000100:00100000:0.0:1766098788.799175:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2065
00000100:00100000:0.0:1766098788.799179:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ad04e0d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432887424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098788.799185:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432887488:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (119us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098788.799194:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 966
00000100:00100000:0.0:1766098788.799211:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ad04e0d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432887424:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (517us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098788.799218:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2065
00010000:00080000:1.0:1766098788.799232:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000dd0a2da1 x1851885432887488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098795 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766098788.799242:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432887488:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098788.799284:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887232
00000100:00100000:1.0:1766098788.799311:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1405
00000100:00100000:1.0:1766098788.799316:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432887232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098788.799352:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad887d7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432887424:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098788.799411:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d35037c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432887296:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098788.799419:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432887232:12345-0@lo:400:kworker/u8:3.0 Request processed in 101us (424us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098788.799428:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1405
00000100:00100000:0.0:1766098788.799442:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432887232:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098789.245746:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098790.271095:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098791.295583:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098792.321820:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098793.248939:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098793.249344:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098793.249397:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098793.249426:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194304
00000100:00100000:2.0:1766098793.249503:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2066
00000100:00100000:2.0:1766098793.249510:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a951f581 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194304:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098793.249579:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194368
00000100:00100000:2.0:1766098793.249588:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a951f581 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194304:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (244us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098793.249596:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2066
00000100:00100000:3.0:1766098793.249621:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1259
00000100:00100000:3.0:1766098793.249627:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417194368:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098793.249762:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098793.249811:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417194368:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 183us (1840us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098793.249818:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1259
00000100:00100000:0.0:1766098793.250159:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194240
00000100:00100000:0.0:1766098793.250200:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1406
00000100:00100000:0.0:1766098793.250205:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194240:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098793.250250:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194240:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 44us (854us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098793.250257:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1406
00010000:00010000:2.0:1766098793.341754:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098793.917788:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098793.917796:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098793.917814:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098793.917821:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3297
00000100:00080000:0.0:1766098793.917824:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766098793.917829:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098793.917864:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006d35037c x1851885432887680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098793.917954:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432887616:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098793.918013:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098793.918046:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098793.918051:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad887d7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432887744:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098793.918075:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098793.918109:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d35037c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432887680:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098793.918135:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098793.918410:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432887552:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098793.918442:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098793.918469:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098793.918500:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432887808:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098793.918525:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098793.918668:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887680
00000100:00100000:3.0:1766098793.918708:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887808
00000100:00100000:3.0:1766098793.918736:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 967
00000100:00100000:3.0:1766098793.918742:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432887808:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098793.918792:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432887808:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (266us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098793.918798:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 967
00000100:00100000:3.0:1766098793.918815:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1260
00000100:00100000:3.0:1766098793.918819:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432887680:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766098793.918908:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098793.918940:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432887680:12345-0@lo:8:kworker/u8:3.0 Request processed in 120us (805us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098793.918946:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1260
00010000:00080000:3.0:1766098793.918980:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000dd0a2da1 x1851885432887808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098800 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098793.918990:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432887808:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766098793.919019:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098793.919022:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098793.919025:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d35037c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432887680:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098793.919420:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887616
00000100:00100000:2.0:1766098793.919450:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887744
00000100:00100000:2.0:1766098793.919468:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2067
00000100:00100000:2.0:1766098793.919800:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c1ee83e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432887616:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098793.919851:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c1ee83e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432887616:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (1838us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098793.919872:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2067
00000100:00100000:2.0:1766098793.919885:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2068
00000100:00100000:2.0:1766098793.919888:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ad805d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432887744:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098793.919917:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ad805d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432887744:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (1841us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098793.919922:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2068
00000100:00100000:1.0:1766098793.920052:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad887d7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432887744:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098793.920065:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432887616:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098793.921781:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887552
00000100:00100000:0.0:1766098793.921825:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1407
00000100:00100000:0.0:1766098793.921831:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432887552:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098793.921918:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432887552:12345-0@lo:400:kworker/u8:3.0 Request processed in 86us (3475us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098793.921926:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1407
00000100:00100000:2.0:1766098793.922278:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432887552:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098794.365661:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098795.389797:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098796.414980:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098797.438657:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098798.365576:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098798.365622:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098798.365751:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194432
00000100:00100000:0.0:1766098798.365802:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1408
00000100:00100000:0.0:1766098798.365807:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417194432:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098798.365872:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417194432:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 64us (304us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098798.365892:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1408
00000100:00100000:2.0:1766098798.366584:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098798.366627:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194560
00000100:00100000:1.0:1766098798.366660:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1261
00000100:00100000:1.0:1766098798.366673:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034f090f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417194560:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098798.366765:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098798.366812:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034f090f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417194560:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 147us (229us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098798.366819:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1261
00000100:00100000:3.0:1766098798.367116:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194496
00000100:00100000:3.0:1766098798.367162:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2069
00000100:00100000:3.0:1766098798.367167:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2734238 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417194496:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098798.368761:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2734238 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417194496:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1585us (3131us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098798.368774:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2069
00010000:00010000:2.0:1766098798.461652:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098799.039796:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098799.039804:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098799.039807:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098799.039814:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3302
00000100:00080000:2.0:1766098799.039817:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766098799.039823:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098799.039826:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432887872:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098799.039846:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007aef029b x1851885432888000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098799.039896:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098799.039898:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432888064:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098799.039932:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098799.039938:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005001f4c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432888128:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098799.039948:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098799.039971:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098799.039974:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098799.039994:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee6355f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432887936:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098799.040029:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888064
00000100:00100000:2.0:1766098799.040036:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098799.040074:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887936
00000100:00100000:1.0:1766098799.040185:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432887872
00000100:00100000:1.0:1766098799.040216:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1409
00000100:00100000:1.0:1766098799.040221:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432887872:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098799.040419:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2070
00000100:00100000:2.0:1766098799.040430:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000483c3824 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432888064:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098799.040430:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2071
00000100:00100000:3.0:1766098799.040438:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432887936:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098799.040504:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000483c3824 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432888064:12345-0@lo:400:kworker/u8:3.0 Request processed in 78us (557us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098799.040512:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2070
00000100:00100000:3.0:1766098799.040520:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432887936:12345-0@lo:400:kworker/u8:3.0 Request processed in 82us (484us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098799.040527:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2071
00000100:00100000:3.0:1766098799.040555:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007aef029b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888000:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098799.040556:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888128
00000100:00100000:3.0:1766098799.040589:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098799.040590:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 968
00000100:00100000:2.0:1766098799.040596:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c91368f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432888128:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098799.040639:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c91368f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432888128:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (667us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098799.040641:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888000
00000100:00100000:2.0:1766098799.040647:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 968
00000100:00100000:2.0:1766098799.040673:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1262
00000100:00100000:2.0:1766098799.040676:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432888000:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098799.040713:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee6355f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432887936:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098799.040724:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432888064:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766098799.040758:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005001f4c3 x1851885432888128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098806 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098799.040766:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005001f4c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432888128:0@lo:400:kworker/u8:3.0
00010000:02020000:2.0:1766098799.040778:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098799.040815:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432888000:12345-0@lo:8:kworker/u8:3.0 Request processed in 137us (229us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098799.040822:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1262
00000100:00080000:3.0:1766098799.040825:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098799.040830:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098799.040835:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007aef029b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888000:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098799.045306:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432887872:12345-0@lo:400:kworker/u8:3.0 Request processed in 5077us (5404us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098799.045321:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1409
00000100:00100000:3.0:1766098799.045424:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432887872:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098799.485794:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098800.509903:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098801.535326:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098802.558688:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098803.484443:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098803.484548:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098803.484741:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194688
00000100:00100000:0.0:1766098803.484862:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2072
00000100:00100000:0.0:1766098803.484869:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047eeb220 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194688:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098803.484937:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047eeb220 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194688:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 68us (503us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098803.484944:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2072
00000100:00100000:1.0:1766098803.485009:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194624
00000100:00100000:1.0:1766098803.485041:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1410
00000100:00100000:1.0:1766098803.485045:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194624:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098803.485067:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194624:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 22us (521us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098803.485072:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1410
00000100:00100000:2.0:1766098803.487545:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098803.487757:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194752
00000100:00100000:0.0:1766098803.487803:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1263
00000100:00100000:0.0:1766098803.487809:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417194752:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098803.487903:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098803.487963:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417194752:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 153us (426us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098803.487971:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1263
00010000:00010000:0.0:1766098803.581861:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098804.159265:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098804.159271:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766098804.159274:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098804.159279:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098804.159289:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3307
00000100:00080000:3.0:1766098804.159293:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:2.0:1766098804.159300:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000110634bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432888192:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098804.159301:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098804.159326:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000038aa4304 x1851885432888320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098804.159374:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000973030be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432888256:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098804.159381:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098804.159416:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098804.159434:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d77d75e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432888448:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098804.159444:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098804.159452:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000452fc292 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432888384:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098804.159468:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098804.159516:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098804.159545:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888448
00000100:00100000:3.0:1766098804.159555:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038aa4304 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888320:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098804.159581:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098804.159592:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888320
00000100:00100000:3.0:1766098804.159614:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888192
00000100:00100000:2.0:1766098804.159618:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 969
00000100:00100000:2.0:1766098804.159650:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432888448:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098804.159659:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1411
00000100:00100000:3.0:1766098804.159665:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432888192:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098804.159702:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432888448:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (235us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098804.159711:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 969
00000100:00100000:3.0:1766098804.159714:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432888192:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (333us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098804.159723:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1411
00000100:00100000:2.0:1766098804.159732:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1264
00000100:00100000:2.0:1766098804.159737:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432888320:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098804.159759:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888256
00000100:00100000:3.0:1766098804.159782:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888384
00000100:00100000:3.0:1766098804.159798:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2073
00000100:00100000:3.0:1766098804.159802:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432888256:12345-0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766098804.159832:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098804.159833:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432888256:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (418us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098804.159842:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2073
00000100:00100000:3.0:1766098804.159854:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2074
00000100:00100000:3.0:1766098804.159859:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432888384:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098804.159870:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432888320:12345-0@lo:8:kworker/u8:0.0 Request processed in 131us (289us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098804.159877:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1264
00000100:00100000:3.0:1766098804.159892:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432888384:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (377us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098804.159899:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2074
00010000:00080000:2.0:1766098804.159929:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d77d75e6 x1851885432888448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098811 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098804.159940:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d77d75e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432888448:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098804.159940:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098804.159946:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098804.159952:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038aa4304 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888320:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098804.159956:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000110634bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432888192:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098804.159986:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000452fc292 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432888384:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098804.159998:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000973030be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432888256:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098804.606292:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098805.629720:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098806.656181:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098807.678167:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098808.605624:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098808.607212:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098808.607440:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194880
00000100:00100000:1.0:1766098808.607520:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2075
00000100:00100000:1.0:1766098808.607527:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005001f4c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194880:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098808.607621:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005001f4c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417194880:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (2004us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098808.607629:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2075
00000100:00100000:2.0:1766098808.608875:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098808.609725:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194816
00000100:00100000:0.0:1766098808.609779:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417194944
00000100:00100000:3.0:1766098808.609782:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1412
00000100:00100000:3.0:1766098808.609787:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194816:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098808.609808:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1265
00000100:00100000:0.0:1766098808.609811:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417194944:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:3.0:1766098808.609857:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417194816:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 69us (2652us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098808.609865:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1412
00010000:02020000:0.0:1766098808.609901:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098808.609929:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417194944:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 117us (1060us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098808.609935:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1265
00010000:00010000:2.0:1766098808.703408:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098809.277779:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098809.277786:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098809.277790:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098809.277797:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3312
00000100:00080000:2.0:1766098809.277801:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098809.277807:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098809.277828:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000070167dff x1851885432888640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098809.277934:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432888768:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.277999:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098809.278040:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098809.278046:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee6355f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432888512:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278073:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098809.278101:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432888576:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278126:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098809.278140:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098809.278144:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432888704:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278162:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098809.278185:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888640:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278207:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098809.278239:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888512
00000100:00100000:3.0:1766098809.278277:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1413
00000100:00100000:3.0:1766098809.278283:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432888512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278323:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432888512:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (250us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098809.278331:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1413
00000100:00100000:3.0:1766098809.278379:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee6355f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432888512:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278428:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888576
00000100:00100000:0.0:1766098809.278464:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888768
00000100:00100000:3.0:1766098809.278464:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888704
00000100:00100000:0.0:1766098809.278509:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888640
00000100:00100000:3.0:1766098809.278529:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2076
00000100:00100000:0.0:1766098809.278538:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 970
00000100:00100000:3.0:1766098809.278538:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000452fc292 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432888576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098809.278546:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432888768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278596:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000452fc292 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432888576:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (458us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098809.278604:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2076
00000100:00100000:0.0:1766098809.278609:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432888768:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (609us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098809.278616:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 970
00000100:00100000:3.0:1766098809.278620:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2077
00000100:00100000:3.0:1766098809.278625:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000110634bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432888704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098809.278635:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1266
00000100:00100000:0.0:1766098809.278640:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d77d75e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432888640:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278655:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000110634bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432888704:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (491us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098809.278661:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2077
00000100:00100000:3.0:1766098809.278692:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432888704:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098809.278701:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432888576:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766098809.278739:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:3.0:1766098809.278742:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000369e20f7 x1851885432888768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098816 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098809.278751:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432888768:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098809.278777:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d77d75e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432888640:12345-0@lo:8:kworker/u8:3.0 Request processed in 136us (570us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098809.278784:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1266
00000100:00080000:0.0:1766098809.278809:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098809.278813:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098809.278817:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888640:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766098809.728387:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098810.749723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098811.773764:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098812.798313:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098813.725402:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098813.727105:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098813.727780:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098813.727888:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195136
00000100:00100000:1.0:1766098813.727949:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1267
00000100:00100000:1.0:1766098813.727957:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417195136:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098813.728062:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098813.728147:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417195136:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 189us (368us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098813.728156:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1267
00000100:00100000:2.0:1766098813.728407:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195072
00000100:00100000:2.0:1766098813.728461:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2078
00000100:00100000:2.0:1766098813.728466:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f676fcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417195072:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098813.728532:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f676fcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417195072:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 65us (3144us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098813.728539:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2078
00000100:00100000:3.0:1766098813.728693:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195008
00000100:00100000:3.0:1766098813.728737:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1414
00000100:00100000:3.0:1766098813.728742:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417195008:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098813.728774:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417195008:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 31us (1674us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098813.728780:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1414
00010000:00010000:2.0:1766098813.825964:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098814.398166:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098814.398176:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098814.398181:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098814.398192:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3317
00000100:00080000:0.0:1766098814.398197:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098814.398222:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098814.398251:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000dc68e892 x1851885432888960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098814.398315:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc68e892 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888960:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098814.398399:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098814.398470:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888960
00000100:00100000:0.0:1766098814.398531:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1268
00000100:00100000:0.0:1766098814.398538:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432888960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098814.398657:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098814.398706:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432888960:12345-0@lo:8:kworker/u8:0.0 Request processed in 168us (309us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098814.398714:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1268
00000100:00080000:0.0:1766098814.398764:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098814.398768:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098814.398772:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc68e892 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432888960:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098814.398860:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432888832:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098814.398900:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098814.398931:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098814.398937:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432889088:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098814.398961:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098814.398985:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889088
00000100:00100000:1.0:1766098814.399028:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 971
00000100:00100000:1.0:1766098814.399032:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432889088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098814.399076:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432889088:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (114us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098814.399082:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 971
00010000:00080000:1.0:1766098814.399113:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000daafc58 x1851885432889088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098821 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098814.399125:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432889088:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098814.399212:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7b70f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432888896:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098814.399246:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098814.399287:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098814.399294:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432889024:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098814.399318:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098814.399404:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888896
00000100:00100000:2.0:1766098814.399444:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889024
00000100:00100000:2.0:1766098814.399463:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2079
00000100:00100000:2.0:1766098814.399466:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432888896:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098814.399528:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432888896:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (282us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098814.399534:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2079
00000100:00100000:2.0:1766098814.399556:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2080
00000100:00100000:2.0:1766098814.399560:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432889024:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098814.399592:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432889024:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (274us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098814.399598:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2080
00000100:00100000:3.0:1766098814.400246:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432889024:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098814.400259:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7b70f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432888896:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098814.400725:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432888832
00000100:00100000:0.0:1766098814.400774:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1415
00000100:00100000:0.0:1766098814.400779:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432888832:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098814.400857:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432888832:12345-0@lo:400:kworker/u8:0.0 Request processed in 77us (1956us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098814.400865:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1415
00000100:00100000:1.0:1766098814.401016:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432888832:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098814.847494:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098815.869933:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098816.893748:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098817.919249:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098818.853102:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098818.853854:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098818.853894:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195264
00000100:00100000:1.0:1766098818.853958:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2081
00000100:00100000:1.0:1766098818.853967:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee6355f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417195264:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098818.853983:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195200
00000100:00100000:3.0:1766098818.854001:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098818.854016:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1416
00000100:00100000:0.0:1766098818.854021:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417195200:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098818.854043:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee6355f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417195264:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 76us (947us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098818.854052:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2081
00000100:00100000:0.0:1766098818.854101:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417195200:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 78us (248us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098818.854109:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1416
00000100:00100000:2.0:1766098818.854135:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195328
00000100:00100000:2.0:1766098818.854167:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1269
00000100:00100000:2.0:1766098818.854172:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417195328:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766098818.854265:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098818.854308:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417195328:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 135us (308us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098818.854314:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1269
00010000:00010000:0.0:1766098818.943232:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098819.517795:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098819.517801:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098819.517803:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098819.517810:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3322
00000100:00080000:2.0:1766098819.517813:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098819.517817:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098819.517834:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000070167dff x1851885432889280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098819.517870:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432889280:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098819.517924:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098819.517962:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889280
00000100:00100000:2.0:1766098819.517998:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1270
00000100:00100000:2.0:1766098819.518015:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432889280:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766098819.518121:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098819.518164:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432889280:12345-0@lo:8:kworker/u8:3.0 Request processed in 148us (239us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098819.518171:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1270
00000100:00080000:2.0:1766098819.518207:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098819.518210:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098819.518213:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432889280:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098819.518276:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432889152:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098819.518309:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098819.518335:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098819.518340:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432889408:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098819.518359:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098819.518381:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889408
00000100:00100000:1.0:1766098819.518412:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 972
00000100:00100000:1.0:1766098819.518416:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9e3df9b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432889408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098819.518453:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9e3df9b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432889408:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (93us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098819.518465:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 972
00010000:00080000:1.0:1766098819.518618:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000319ec734 x1851885432889408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098826 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766098819.518628:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432889408:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098819.518749:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432889216:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098819.518778:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098819.518807:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098819.518812:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432889344:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098819.518831:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098819.518863:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889216
00000100:00100000:0.0:1766098819.518864:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889344
00000100:00100000:0.0:1766098819.518891:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2082
00000100:00100000:0.0:1766098819.518895:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432889344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098819.518898:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2083
00000100:00100000:3.0:1766098819.518901:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b47533c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432889216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098819.518927:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432889344:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (96us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098819.518927:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b47533c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432889216:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (150us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098819.518934:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2082
00000100:00100000:3.0:1766098819.518936:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2083
00000100:00100000:3.0:1766098819.518961:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432889344:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098819.518970:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432889216:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098819.519242:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889152
00000100:00100000:2.0:1766098819.519274:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1417
00000100:00100000:2.0:1766098819.519278:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432889152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098819.519311:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432889152:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (1003us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098819.519317:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1417
00000100:00100000:1.0:1766098819.519531:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432889152:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098819.965720:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098820.990569:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098822.013764:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098823.046720:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098823.964636:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098823.964731:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195520
00000100:00100000:3.0:1766098823.964787:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1271
00000100:00100000:3.0:1766098823.964794:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417195520:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098823.964914:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098823.964984:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417195520:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 188us (356us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098823.964992:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1271
00000100:00100000:1.0:1766098823.969533:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098823.970048:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098823.970737:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195456
00000100:00100000:2.0:1766098823.970786:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2084
00000100:00100000:2.0:1766098823.970793:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000677df168 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417195456:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098823.970867:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000677df168 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417195456:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 74us (1341us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098823.970875:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2084
00000100:00100000:0.0:1766098823.973268:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195392
00000100:00100000:0.0:1766098823.973332:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1418
00000100:00100000:0.0:1766098823.973339:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417195392:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098823.973406:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417195392:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 66us (3359us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098823.973413:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1418
00010000:00010000:2.0:1766098824.072577:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098824.637746:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098824.637763:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098824.637766:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098824.637773:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3327
00000100:00080000:2.0:1766098824.637777:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098824.637782:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098824.637801:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e3400fcc x1851885432889600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098824.637843:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432889600:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098824.637905:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098824.637952:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889600
00000100:00100000:2.0:1766098824.637981:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1272
00000100:00100000:2.0:1766098824.637986:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432889600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098824.638087:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098824.638123:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432889600:12345-0@lo:8:kworker/u8:0.0 Request processed in 136us (218us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098824.638129:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1272
00000100:00080000:2.0:1766098824.638171:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098824.638175:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098824.638178:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432889600:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098824.638244:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432889728:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098824.638272:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098824.638297:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098824.638302:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432889472:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098824.638326:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098824.638352:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889472
00000100:00100000:1.0:1766098824.638376:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1419
00000100:00100000:1.0:1766098824.638380:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432889472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098824.638415:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432889472:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (89us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098824.638420:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1419
00000100:00100000:1.0:1766098824.638450:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432889472:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.641507:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432889536:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.641580:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098824.641619:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098824.641627:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432889664:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.641649:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098824.641688:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889728
00000100:00100000:3.0:1766098824.641727:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 973
00000100:00100000:3.0:1766098824.641732:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432889728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.641780:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432889728:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (3507us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098824.641787:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 973
00000100:00100000:3.0:1766098824.641811:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889536
00000100:00100000:3.0:1766098824.641833:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889664
00000100:00100000:3.0:1766098824.641853:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2085
00000100:00100000:3.0:1766098824.641856:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9a5f5e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432889536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.641895:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9a5f5e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432889536:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (316us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098824.641901:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2085
00000100:00100000:3.0:1766098824.641914:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2086
00000100:00100000:3.0:1766098824.641918:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000766b1095 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432889664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.641942:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000766b1095 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432889664:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (293us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098824.641961:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2086
00000100:00100000:3.0:1766098824.642004:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432889664:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098824.642015:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432889536:0@lo:400:kworker/u8:0.0
00010000:00080000:1.0:1766098824.642356:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000df6e7141 x1851885432889728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098831 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098824.642368:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432889728:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098825.086102:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098826.109745:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098827.133788:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098828.159278:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098829.086582:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098829.087155:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098829.087202:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098829.087298:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195648
00000100:00100000:1.0:1766098829.087303:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195584
00000100:00100000:0.0:1766098829.087380:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2087
00000100:00100000:1.0:1766098829.087391:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1420
00000100:00100000:0.0:1766098829.087392:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b219b5c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417195648:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098829.087403:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417195584:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098829.087512:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195712
00000100:00100000:2.0:1766098829.087545:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1273
00000100:00100000:0.0:1766098829.087551:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b219b5c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417195648:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 156us (975us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098829.087557:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417195712:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:0.0:1766098829.087576:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2087
00010000:02020000:2.0:1766098829.087652:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098829.087860:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417195584:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 460us (658us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098829.087868:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1420
00000100:00100000:2.0:1766098829.088552:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417195712:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 994us (1398us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098829.088561:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1273
00010000:00010000:0.0:1766098829.181756:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098829.761654:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098829.761662:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098829.761665:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098829.761672:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3333
00000100:00080000:1.0:1766098829.761676:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098829.761681:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098829.761702:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000aaed12d3 x1851885432889920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098829.761754:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432889984:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098829.761815:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098829.761845:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098829.761851:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432889856:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098829.761873:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098829.761898:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aaed12d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432889920:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098829.761923:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098829.761969:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889984
00000100:00100000:2.0:1766098829.762439:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432890048:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098829.762496:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098829.762517:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098829.762523:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432889792:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098829.762550:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098829.762600:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889920
00000100:00100000:2.0:1766098829.762628:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890048
00000100:00100000:2.0:1766098829.762652:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 974
00000100:00100000:2.0:1766098829.762657:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432890048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098829.763122:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432890048:12345-0@lo:400:kworker/u8:0.0 Request processed in 463us (645us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098829.763130:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 974
00000100:00100000:2.0:1766098829.763151:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1274
00000100:00100000:2.0:1766098829.763154:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432889920:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098829.763251:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098829.763286:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432889920:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (1363us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098829.763292:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1274
00010000:00080000:2.0:1766098829.763325:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ecf17fc2 x1851885432890048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098836 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098829.763337:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432890048:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098829.764513:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889856
00000100:00080000:0.0:1766098829.764715:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098829.764724:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098829.764732:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aaed12d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432889920:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098829.764775:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432889792
00000100:00100000:0.0:1766098829.764830:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1421
00000100:00100000:0.0:1766098829.764837:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432889792:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098829.764849:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2088
00000100:00100000:1.0:1766098829.764855:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432889984:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098829.765096:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432889984:12345-0@lo:400:kworker/u8:0.0 Request processed in 237us (3279us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098829.765116:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2088
00000100:00100000:2.0:1766098829.765235:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2089
00000100:00100000:2.0:1766098829.765240:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9f903ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432889856:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098829.765281:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9f903ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432889856:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (3407us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098829.765287:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2089
00000100:00100000:2.0:1766098829.765365:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432889792:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098829.765523:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432889856:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098829.765534:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432889984:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098829.765799:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432889792:12345-0@lo:400:kworker/u8:0.0 Request processed in 958us (3244us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098829.765810:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1421
00010000:00010000:2.0:1766098830.205744:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098831.229704:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098832.253677:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098833.278759:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098834.205383:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098834.205459:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098834.205672:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195776
00000100:00100000:0.0:1766098834.205750:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1422
00000100:00100000:0.0:1766098834.205758:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417195776:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098834.205848:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417195776:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 89us (476us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098834.205856:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1422
00000100:00100000:2.0:1766098834.207286:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195840
00000100:00100000:2.0:1766098834.207368:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2090
00000100:00100000:2.0:1766098834.207375:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007331255b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417195840:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098834.207490:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007331255b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417195840:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 100us (2017us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098834.207497:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2090
00000100:00100000:0.0:1766098834.209955:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098834.210059:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195904
00000100:00100000:3.0:1766098834.210117:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1275
00000100:00100000:3.0:1766098834.210125:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417195904:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098834.210237:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098834.211241:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417195904:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1111us (1294us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098834.211255:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1275
00010000:00010000:0.0:1766098834.309920:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098834.877963:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098834.877972:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098834.877976:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098834.877986:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3338
00000100:00080000:3.0:1766098834.877990:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098834.877996:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098834.878019:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b0591111 x1851885432890240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098834.878062:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432890304:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098834.878156:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098834.878198:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098834.878203:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432890176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098834.878237:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098834.878276:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890304
00000100:00100000:3.0:1766098834.878318:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890176
00000100:00100000:3.0:1766098834.878336:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2091
00000100:00100000:3.0:1766098834.878342:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432890304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098834.878388:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432890304:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (233us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098834.878405:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2091
00000100:00100000:3.0:1766098834.878423:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2092
00000100:00100000:3.0:1766098834.878426:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432890176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098834.878447:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432890176:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (210us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098834.878452:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2092
00000100:00100000:3.0:1766098834.878508:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432890176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098834.878517:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432890304:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098834.878587:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432890368:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098834.878621:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098834.878647:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098834.878652:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa6f101e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432890112:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098834.878674:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098834.878707:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890112
00000100:00100000:2.0:1766098834.878730:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1423
00000100:00100000:2.0:1766098834.878733:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432890112:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098834.878760:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432890112:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (86us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098834.878766:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1423
00000100:00100000:2.0:1766098834.878804:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa6f101e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432890112:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098834.879047:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432890240:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098834.879076:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098834.879108:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890368
00000100:00100000:0.0:1766098834.879136:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890240
00000100:00100000:0.0:1766098834.879149:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 975
00000100:00100000:0.0:1766098834.879152:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432890368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098834.879180:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432890368:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (559us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098834.879185:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 975
00000100:00100000:0.0:1766098834.879197:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1276
00000100:00100000:0.0:1766098834.879200:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432890240:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098834.879308:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098834.879341:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432890240:12345-0@lo:8:kworker/u8:3.0 Request processed in 137us (263us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098834.879347:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1276
00000100:00080000:0.0:1766098834.879374:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098834.879378:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098834.879381:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432890240:0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766098834.879732:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c2dfefc2 x1851885432890368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098841 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766098834.879742:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432890368:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098835.328596:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098836.349769:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098837.373752:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098838.400047:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098839.330206:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098839.331852:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098839.331941:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417195968
00000100:00100000:0.0:1766098839.331955:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098839.331996:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1424
00000100:00100000:2.0:1766098839.332004:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417195968:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098839.332072:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417195968:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 67us (1874us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098839.332080:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1424
00000100:00100000:1.0:1766098839.332200:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196032
00000100:00100000:1.0:1766098839.332249:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2093
00000100:00100000:1.0:1766098839.332254:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c488766 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417196032:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098839.332302:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c488766 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417196032:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 47us (348us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098839.332308:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2093
00000100:00100000:3.0:1766098839.332732:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196096
00000100:00100000:3.0:1766098839.332801:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1277
00000100:00100000:3.0:1766098839.332805:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004ba22e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417196096:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098839.332910:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098839.334165:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004ba22e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417196096:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1353us (2314us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098839.334195:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1277
00010000:00010000:2.0:1766098839.424525:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098839.997707:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432890432:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098839.997782:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098839.997808:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432890496:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098839.997846:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098839.997851:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890432
00000100:00100000:3.0:1766098839.997891:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1425
00000100:00100000:3.0:1766098839.997898:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432890432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098839.997942:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432890432:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (162us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098839.997950:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1425
00000100:00100000:3.0:1766098839.997990:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432890432:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098839.998018:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890496
00000100:00100000:3.0:1766098839.998048:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2094
00000100:00100000:3.0:1766098839.998052:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432890496:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098839.998250:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098839.998255:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098839.998258:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098839.998263:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3343
00000100:00080000:1.0:1766098839.998265:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098839.998270:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098839.998292:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ffb8f463 x1851885432890560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098839.998379:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4d173d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432890624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098839.998409:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098839.998439:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890624
00000100:00100000:0.0:1766098839.998462:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2095
00000100:00100000:0.0:1766098839.998466:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007626dec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432890624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098840.000780:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432890496:12345-0@lo:400:kworker/u8:0.0 Request processed in 2720us (2928us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098840.000794:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2094
00000100:00100000:1.0:1766098840.000839:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432890560:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098840.000855:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432890688:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098840.002097:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432890496:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098840.002119:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098840.002175:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890688
00000100:00100000:3.0:1766098840.002219:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 976
00000100:00100000:3.0:1766098840.002225:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432890688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098840.002267:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432890688:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (150us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098840.002275:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 976
00010000:00080000:3.0:1766098840.002321:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004584cb44 x1851885432890688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098847 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098840.002331:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432890688:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098840.002559:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007626dec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432890624:12345-0@lo:400:kworker/u8:0.0 Request processed in 4090us (4148us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098840.002569:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2095
00000100:00100000:3.0:1766098840.002590:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4d173d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432890624:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098840.002646:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098840.002727:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890560
00000100:00100000:3.0:1766098840.002757:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1278
00000100:00100000:3.0:1766098840.002761:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009bbfcf2c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432890560:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098840.002859:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098840.002901:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009bbfcf2c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432890560:12345-0@lo:8:kworker/u8:0.0 Request processed in 139us (255us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098840.002908:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1278
00000100:00080000:1.0:1766098840.005462:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098840.005494:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098840.005502:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432890560:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766098840.445942:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098841.469776:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098842.493685:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098843.517702:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098844.447935:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098844.447984:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098844.448072:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098844.448560:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196288
00000100:00100000:3.0:1766098844.448608:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1279
00000100:00100000:3.0:1766098844.448615:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417196288:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098844.448712:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098844.448774:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417196288:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 158us (702us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098844.448781:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1279
00000100:00100000:1.0:1766098844.449574:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196224
00000100:00100000:1.0:1766098844.449691:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2096
00000100:00100000:1.0:1766098844.449696:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bedb856 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417196224:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098844.449838:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bedb856 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417196224:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 141us (1910us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098844.449846:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2096
00000100:00100000:2.0:1766098844.449894:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196160
00000100:00100000:2.0:1766098844.449936:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1426
00000100:00100000:2.0:1766098844.449940:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417196160:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098844.449965:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417196160:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 25us (1982us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098844.449971:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1426
00010000:00010000:0.0:1766098844.542497:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098845.120365:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098845.120371:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098845.120374:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098845.120381:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3348
00000100:00080000:3.0:1766098845.120383:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098845.120389:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098845.120419:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b0591111 x1851885432890880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098845.120462:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa6f101e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432890944:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098845.120574:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432890752:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098845.120642:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098845.120668:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098845.120672:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006131d46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432891008:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098845.120696:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098845.120727:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891008
00000100:00100000:0.0:1766098845.120764:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 977
00000100:00100000:0.0:1766098845.120769:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432891008:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098845.120816:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432891008:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (119us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098845.120822:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 977
00010000:00080000:0.0:1766098845.120856:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000006131d46 x1851885432891008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098852 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766098845.120869:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006131d46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432891008:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098845.120942:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432890816:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098845.120970:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098845.121063:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432890880:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098845.121091:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098845.121131:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890752
00000100:00100000:1.0:1766098845.121153:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1427
00000100:00100000:1.0:1766098845.121156:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432890752:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098845.121190:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432890752:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (549us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098845.121195:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1427
00000100:00100000:1.0:1766098845.121224:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432890752:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098845.124428:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890880
00000100:00100000:2.0:1766098845.124509:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1280
00000100:00100000:2.0:1766098845.124515:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432890880:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766098845.124611:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098845.124690:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432890880:12345-0@lo:8:kworker/u8:3.0 Request processed in 174us (3598us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098845.124697:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1280
00000100:00080000:2.0:1766098845.124742:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098845.124746:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098845.124750:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432890880:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098845.124857:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890816
00000100:00100000:0.0:1766098845.124897:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2097
00000100:00100000:0.0:1766098845.124914:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432890816:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098845.124960:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432890816:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (3989us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098845.124966:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2097
00000100:00100000:3.0:1766098845.125230:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098845.125312:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432890944
00000100:00100000:2.0:1766098845.125353:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2098
00000100:00100000:2.0:1766098845.125357:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ebb20fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432890944:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098845.125401:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ebb20fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432890944:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (172us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098845.125407:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2098
00000100:00100000:2.0:1766098845.125439:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432890816:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098845.125511:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa6f101e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432890944:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098845.567865:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098846.592721:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098847.613666:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098848.639013:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098849.568335:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098849.568384:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098849.568419:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196416
00000100:00100000:2.0:1766098849.568469:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2099
00000100:00100000:2.0:1766098849.568501:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3e125c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417196416:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098849.568592:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3e125c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417196416:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 112us (260us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098849.568600:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2099
00000100:00100000:3.0:1766098849.569259:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098849.569337:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196352
00000100:00100000:1.0:1766098849.569383:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1428
00000100:00100000:1.0:1766098849.569388:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417196352:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098849.569436:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417196352:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 47us (1052us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098849.569443:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1428
00000100:00100000:0.0:1766098849.569633:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196480
00000100:00100000:0.0:1766098849.570185:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1281
00000100:00100000:0.0:1766098849.570190:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000632c907e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417196480:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766098849.570281:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098849.571187:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000632c907e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417196480:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 994us (1927us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098849.571197:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1281
00010000:00010000:0.0:1766098849.661766:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098850.239353:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098850.240049:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432891072:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098850.240120:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098850.240240:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891072
00000100:00100000:3.0:1766098850.240296:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1429
00000100:00100000:3.0:1766098850.240302:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432891072:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098850.240346:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432891072:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (226us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098850.240352:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1429
00000100:00100000:1.0:1766098850.240438:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432891072:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098850.240663:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766098850.241375:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432891136:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098850.241408:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098850.241509:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891136
00000100:00100000:3.0:1766098850.241534:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2100
00000100:00100000:3.0:1766098850.241538:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4d173d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432891136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098850.241584:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4d173d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432891136:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (176us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098850.241590:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2100
00000100:00100000:1.0:1766098850.241658:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432891136:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098850.245866:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098850.245877:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098850.245880:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098850.245887:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3353
00000100:00080000:2.0:1766098850.245890:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098850.245895:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098850.245931:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000061093366 x1851885432891200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098850.246017:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098850.246033:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432891328:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098850.246094:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098850.246146:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891328
00000100:00100000:2.0:1766098850.246188:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 978
00000100:00100000:2.0:1766098850.246194:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432891328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098850.246239:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432891328:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (145us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098850.246247:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 978
00010000:00080000:2.0:1766098850.246299:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008d577608 x1851885432891328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098857 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098850.246311:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432891328:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098850.246392:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432891200:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098850.246420:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098850.246612:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432891264:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098850.246644:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098850.246720:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891264
00000100:00100000:1.0:1766098850.246753:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2101
00000100:00100000:1.0:1766098850.246757:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000053dd58eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432891264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098850.246790:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000053dd58eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432891264:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (147us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098850.246796:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2101
00000100:00100000:3.0:1766098850.246862:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432891264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098850.246974:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891200
00000100:00100000:2.0:1766098850.247005:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1282
00000100:00100000:2.0:1766098850.247008:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432891200:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098850.247096:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098850.247141:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432891200:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (720us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098850.247162:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1282
00000100:00080000:1.0:1766098850.247230:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098850.247234:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098850.247238:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432891200:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766098850.685678:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098851.709786:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098852.735844:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098853.757667:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098854.688134:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098854.688193:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098854.689062:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196672
00000100:00100000:3.0:1766098854.690566:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098854.690868:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1283
00000100:00100000:1.0:1766098854.690878:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417196672:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766098854.690979:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098854.691305:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196544
00000100:00100000:0.0:1766098854.691360:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1430
00000100:00100000:0.0:1766098854.691365:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417196544:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098854.691433:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417196544:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 67us (3240us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098854.691442:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1430
00000100:00100000:3.0:1766098854.693755:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196608
00000100:00100000:3.0:1766098854.693895:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2102
00000100:00100000:3.0:1766098854.693902:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006131d46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417196608:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766098854.693999:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006131d46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417196608:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 96us (3442us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098854.694007:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2102
00000100:00100000:1.0:1766098854.695130:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417196672:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 4246us (7002us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098854.695156:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1283
00010000:00010000:2.0:1766098854.782927:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098855.358170:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432891392:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098855.358343:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098855.358442:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891392
00000100:00100000:3.0:1766098855.358524:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1431
00000100:00100000:3.0:1766098855.358532:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432891392:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766098855.360105:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098855.360114:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098855.360134:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098855.360142:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3358
00000100:00080000:0.0:1766098855.360146:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098855.360152:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098855.360182:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c63c9d57 x1851885432891520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098855.361181:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432891392:12345-0@lo:400:kworker/u8:3.0 Request processed in 2643us (2835us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098855.361197:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1431
00000100:00100000:3.0:1766098855.361280:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432891584:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361343:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098855.361377:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098855.361383:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea8a9953 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432891456:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361406:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098855.361444:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891584
00000100:00100000:3.0:1766098855.361510:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891456
00000100:00100000:3.0:1766098855.361532:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2103
00000100:00100000:3.0:1766098855.361538:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432891584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361597:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432891584:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (254us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098855.361604:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2103
00000100:00100000:3.0:1766098855.361618:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2104
00000100:00100000:3.0:1766098855.361622:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041ad3224 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432891456:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361662:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041ad3224 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432891456:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (256us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098855.361668:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2104
00000100:00100000:3.0:1766098855.361712:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea8a9953 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432891456:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361721:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432891584:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361866:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432891648:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361899:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098855.361939:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432891520:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098855.361965:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098855.361997:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891648
00000100:00100000:3.0:1766098855.362023:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891520
00000100:00100000:3.0:1766098855.362038:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 979
00000100:00100000:3.0:1766098855.362042:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432891648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098855.362075:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432891648:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (177us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098855.362081:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 979
00000100:00100000:3.0:1766098855.362092:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1284
00000100:00100000:3.0:1766098855.362096:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432891520:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766098855.362205:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098855.362239:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432891520:12345-0@lo:8:kworker/u8:3.0 Request processed in 142us (274us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098855.362258:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1284
00000100:00080000:3.0:1766098855.362283:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098855.362287:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098855.362298:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432891520:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766098855.362323:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004722f0a4 x1851885432891648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098862 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098855.362335:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432891648:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098855.363794:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432891392:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098855.806190:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098856.829673:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098857.854408:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098858.878456:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098859.805130:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098859.805534:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196736
00000100:00100000:2.0:1766098859.805608:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1432
00000100:00100000:2.0:1766098859.805617:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417196736:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098859.805725:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417196736:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 107us (608us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098859.805733:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1432
00000100:00100000:3.0:1766098859.811369:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098859.812566:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196800
00000100:00100000:0.0:1766098859.812652:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2105
00000100:00100000:0.0:1766098859.812661:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417196800:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098859.812790:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417196800:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 129us (1432us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098859.812800:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2105
00000100:00100000:3.0:1766098859.815710:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098859.815844:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196864
00000100:00100000:1.0:1766098859.815906:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1285
00000100:00100000:1.0:1766098859.815915:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417196864:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098859.816020:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098859.816117:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417196864:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 202us (418us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098859.816126:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1285
00010000:00010000:0.0:1766098859.902611:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098860.477956:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098860.478802:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432891776:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098860.478892:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766098860.478946:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098860.478952:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098860.478956:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098860.478962:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3363
00000100:00080000:2.0:1766098860.478965:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:0.0:1766098860.479104:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432891712:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098860.479145:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098860.479190:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891712
00000100:00100000:0.0:1766098860.479259:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1433
00000100:00100000:0.0:1766098860.479267:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432891712:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098860.479324:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432891712:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (178us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098860.479333:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1433
00000100:00100000:0.0:1766098860.479389:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432891712:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766098860.479713:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098860.479751:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000070167dff x1851885432891840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098860.479839:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098860.479847:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432891904:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098860.479881:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098860.482777:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098860.482793:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891776
00000100:00100000:3.0:1766098860.482800:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432891968:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098860.482864:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891904
00000100:00100000:3.0:1766098860.482885:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098860.482904:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2106
00000100:00100000:2.0:1766098860.482915:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432891776:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098860.482951:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891968
00000100:00100000:3.0:1766098860.483008:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980
00000100:00100000:3.0:1766098860.483023:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432891968:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098860.483027:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432891776:12345-0@lo:400:kworker/u8:3.0 Request processed in 103us (4126us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098860.483043:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2106
00000100:00100000:2.0:1766098860.483111:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2107
00000100:00100000:3.0:1766098860.483114:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432891968:12345-0@lo:400:kworker/u8:3.0 Request processed in 87us (220us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098860.483121:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432891904:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098860.483138:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980
00000100:00100000:2.0:1766098860.483181:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432891904:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (3297us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098860.483190:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2107
00010000:00080000:3.0:1766098860.483199:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f7b9f638 x1851885432891968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098867 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098860.483226:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432891968:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098860.483270:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432891904:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098860.483295:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432891776:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098860.486338:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432891840:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098860.486442:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098860.487192:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432891840
00000100:00100000:1.0:1766098860.487257:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1286
00000100:00100000:1.0:1766098860.487265:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7b70f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432891840:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766098860.487377:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098860.487445:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7b70f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432891840:12345-0@lo:8:kworker/u8:3.0 Request processed in 179us (1004us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098860.487453:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1286
00000100:00080000:0.0:1766098860.487676:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098860.487681:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098860.487686:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432891840:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766098860.925750:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098861.949700:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098862.976808:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098863.999930:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098864.926825:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098864.926874:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098864.926967:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098864.927341:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196992
00000100:00100000:0.0:1766098864.927398:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2108
00000100:00100000:0.0:1766098864.927405:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417196992:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766098864.927472:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417196992:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 67us (655us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098864.927509:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2108
00000100:00100000:2.0:1766098864.927590:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417196928
00000100:00100000:2.0:1766098864.927628:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1434
00000100:00100000:2.0:1766098864.927632:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417196928:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098864.927661:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417196928:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 28us (695us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098864.927667:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1434
00000100:00100000:3.0:1766098864.927801:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197056
00000100:00100000:3.0:1766098864.927838:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1287
00000100:00100000:3.0:1766098864.927842:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417197056:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098864.927942:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098864.927972:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417197056:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 129us (1099us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098864.927989:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1287
00010000:00010000:0.0:1766098865.024348:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098865.597924:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098865.597930:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098865.597934:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098865.597943:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3368
00000100:00080000:2.0:1766098865.597946:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098865.597951:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098865.597972:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000070167dff x1851885432892160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098865.598001:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432892096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098865.598064:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098865.598106:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098865.598111:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432892224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098865.598133:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000377acd23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432892288:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098865.598134:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098865.598191:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098865.598209:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098865.598214:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432892032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098865.598236:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098865.598264:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892096
00000100:00100000:2.0:1766098865.598264:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892288
00000100:00100000:1.0:1766098865.598296:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892224
00000100:00100000:2.0:1766098865.598298:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 981
00000100:00100000:2.0:1766098865.598304:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432892288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098865.598311:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2109
00000100:00100000:1.0:1766098865.598313:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432892096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098865.598355:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432892096:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (292us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098865.598362:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2109
00000100:00100000:1.0:1766098865.598376:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2110
00000100:00100000:1.0:1766098865.598379:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432892224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098865.598402:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432892224:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (268us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098865.598406:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2110
00000100:00100000:1.0:1766098865.598434:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432892224:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098865.598442:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432892096:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098865.598632:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432892160:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098865.598663:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098865.598961:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892032
00000100:00100000:0.0:1766098865.598989:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1435
00000100:00100000:0.0:1766098865.598992:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432892032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098865.599028:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432892032:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (791us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098865.599034:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1435
00000100:00100000:0.0:1766098865.599072:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432892032:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098865.599277:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892160
00000100:00100000:3.0:1766098865.599308:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1288
00000100:00100000:3.0:1766098865.599311:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432892160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766098865.599425:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098865.599459:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432892160:12345-0@lo:8:kworker/u8:3.0 Request processed in 147us (796us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098865.599465:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1288
00000100:00080000:3.0:1766098865.599504:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098865.599507:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098865.599510:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070167dff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432892160:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098865.599768:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432892288:12345-0@lo:400:kworker/u8:3.0 Request processed in 1464us (1577us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098865.599775:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 981
00010000:00080000:3.0:1766098865.600873:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000377acd23 x1851885432892288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098872 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098865.600885:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000377acd23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432892288:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098866.046261:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098867.070188:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098868.093695:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098869.117795:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098870.045295:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098870.045360:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098870.046039:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197184
00000100:00100000:0.0:1766098870.046129:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2111
00000100:00100000:0.0:1766098870.046149:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d91044b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417197184:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098870.046248:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197120
00000100:00100000:3.0:1766098870.046301:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1436
00000100:00100000:3.0:1766098870.046308:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417197120:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098870.047093:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d91044b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417197184:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 939us (1808us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098870.047101:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417197120:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 787us (1735us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098870.047120:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2111
00000100:00100000:3.0:1766098870.047121:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1436
00000100:00100000:2.0:1766098870.047393:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098870.051148:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197248
00000100:00100000:0.0:1766098870.051258:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1289
00000100:00100000:0.0:1766098870.051268:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417197248:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098870.051373:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098870.051503:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417197248:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 235us (4111us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098870.051511:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1289
00010000:00010000:2.0:1766098870.141856:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098870.717938:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766098870.717943:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:3.0:1766098870.717951:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098870.717957:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766098870.717967:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432892416:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098870.717968:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432892352:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766098870.717982:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3374
00000100:00080000:3.0:1766098870.717986:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098870.717994:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098870.718026:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e4dd01a1 x1851885432892480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098870.718074:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098870.718079:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432892608:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098870.718121:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432892480:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098870.718123:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098870.718146:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098870.718157:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432892544:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098870.718181:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098870.718198:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098870.718210:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892352
00000100:00100000:0.0:1766098870.718230:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892544
00000100:00100000:1.0:1766098870.718257:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892608
00000100:00100000:3.0:1766098870.718266:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1437
00000100:00100000:3.0:1766098870.718275:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432892352:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098870.718278:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2112
00000100:00100000:0.0:1766098870.718287:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070167dff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432892544:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098870.718318:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892480
00000100:00100000:3.0:1766098870.718345:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432892352:12345-0@lo:400:kworker/u8:3.0 Request processed in 69us (271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098870.718349:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 982
00000100:00100000:0.0:1766098870.718352:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070167dff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432892544:12345-0@lo:400:kworker/u8:3.0 Request processed in 63us (154us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098870.718359:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432892608:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098870.718362:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1437
00000100:00100000:0.0:1766098870.718368:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2112
00000100:00100000:1.0:1766098870.718409:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432892608:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (287us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098870.718416:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1290
00000100:00100000:1.0:1766098870.718425:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 982
00000100:00100000:3.0:1766098870.718425:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432892480:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098870.718431:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432892544:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098870.718505:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432892352:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766098870.718556:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098870.718607:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432892480:12345-0@lo:8:kworker/u8:3.0 Request processed in 185us (428us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098870.718616:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1290
00010000:00080000:3.0:1766098870.718650:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000000365d5a x1851885432892608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098877 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098870.718660:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432892608:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098870.718936:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098870.719917:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892416
00000100:00100000:0.0:1766098870.719981:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2113
00000100:00100000:0.0:1766098870.719986:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b413e52a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432892416:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098870.720054:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b413e52a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432892416:12345-0@lo:400:kworker/u8:3.0 Request processed in 66us (1119us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098870.720066:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2113
00000100:00100000:2.0:1766098870.720143:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432892416:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766098870.721748:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098870.721760:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098870.721768:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432892480:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766098871.165755:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098872.190376:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098873.213748:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098874.237683:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098875.165680:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098875.165737:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098875.165859:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197312
00000100:00100000:0.0:1766098875.166753:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1438
00000100:00100000:0.0:1766098875.166766:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417197312:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098875.166888:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417197312:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 124us (1221us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098875.166895:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1438
00000100:00100000:2.0:1766098875.168749:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197376
00000100:00100000:2.0:1766098875.168842:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2114
00000100:00100000:2.0:1766098875.168850:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000319ec734 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417197376:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098875.170128:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000319ec734 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417197376:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1270us (4383us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098875.170145:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2114
00000100:00100000:1.0:1766098875.170522:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098875.171185:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197440
00000100:00100000:0.0:1766098875.171251:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1291
00000100:00100000:0.0:1766098875.171258:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417197440:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098875.171367:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098875.171454:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417197440:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 195us (934us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098875.171462:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1291
00010000:00010000:0.0:1766098875.261827:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098875.837840:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098875.837852:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098875.837857:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098875.837869:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3379
00000100:00080000:1.0:1766098875.837873:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098875.837881:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098875.837913:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000039dc5372 x1851885432892800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098875.838398:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432892800:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098875.838520:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098875.838641:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432892928:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098875.838675:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098875.838727:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098875.838733:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432892672:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098875.838766:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098875.838807:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892800
00000100:00100000:3.0:1766098875.838846:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892928
00000100:00100000:3.0:1766098875.838877:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 983
00000100:00100000:3.0:1766098875.838884:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432892928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098875.838940:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432892928:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (264us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098875.838948:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 983
00000100:00100000:3.0:1766098875.838966:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1292
00000100:00100000:3.0:1766098875.838970:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432892800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098875.839071:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098875.839119:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432892800:12345-0@lo:8:kworker/u8:0.0 Request processed in 148us (601us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098875.839126:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1292
00000100:00100000:2.0:1766098875.839455:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa6f101e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432892864:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098875.839521:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766098875.840457:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098875.840495:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098875.840501:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432892800:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098875.840551:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892672
00000100:00100000:1.0:1766098875.840596:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1439
00000100:00100000:1.0:1766098875.840602:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432892672:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098875.840665:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432892672:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (1898us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098875.840673:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1439
00000100:00100000:2.0:1766098875.840796:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098875.840807:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432892736:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098875.840840:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098875.840909:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432892672:0@lo:400:kworker/u8:0.0
00010000:00080000:2.0:1766098875.840924:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c2dfefc2 x1851885432892928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098882 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098875.840935:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432892928:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098875.840968:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892864
00000100:00100000:2.0:1766098875.840998:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892736
00000100:00100000:2.0:1766098875.841016:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2115
00000100:00100000:2.0:1766098875.841020:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000377acd23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432892864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098875.841057:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000377acd23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432892864:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (1536us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098875.841063:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2115
00000100:00100000:2.0:1766098875.841077:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2116
00000100:00100000:2.0:1766098875.841080:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432892736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098875.841116:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432892736:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (276us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098875.841122:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2116
00000100:00100000:2.0:1766098875.841154:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432892736:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098875.841164:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa6f101e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432892864:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098876.286014:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098877.310739:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098878.333723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098879.357659:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098880.287728:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098880.287851:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098880.288379:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197632
00000100:00100000:1.0:1766098880.288748:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098880.288813:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197504
00000100:00100000:1.0:1766098880.288849:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1440
00000100:00100000:1.0:1766098880.288861:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417197504:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766098880.288935:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417197504:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 73us (192us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098880.288944:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1440
00000100:00100000:2.0:1766098880.288945:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197568
00000100:00100000:2.0:1766098880.289007:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2117
00000100:00100000:2.0:1766098880.289012:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa6f101e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417197568:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766098880.289046:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa6f101e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417197568:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 33us (1196us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098880.289053:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2117
00000100:00100000:3.0:1766098880.289166:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1293
00000100:00100000:3.0:1766098880.289173:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417197632:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766098880.289272:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098880.289918:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417197632:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 742us (2195us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098880.289926:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1293
00010000:00010000:0.0:1766098880.382879:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098880.957936:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098880.957946:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098880.957965:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098880.957974:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3384
00000100:00080000:3.0:1766098880.957978:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098880.957984:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098880.958013:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005114f03e x1851885432893120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098880.958092:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432893056:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098880.958168:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098880.958204:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098880.958208:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432893184:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098880.958224:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098880.958287:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432893248:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098880.958321:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098880.958346:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098880.958350:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432892992:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098880.958377:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098880.958398:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432893120:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098880.958418:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098880.959021:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893248
00000100:00100000:0.0:1766098880.959087:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893120
00000100:00100000:0.0:1766098880.959108:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984
00000100:00100000:0.0:1766098880.959115:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432893248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098880.959180:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432893248:12345-0@lo:400:kworker/u8:3.0 Request processed in 64us (859us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098880.959188:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984
00000100:00100000:0.0:1766098880.959211:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1294
00000100:00100000:0.0:1766098880.959215:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432893120:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098880.959318:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098880.959337:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893056
00000100:00100000:0.0:1766098880.959358:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432893120:12345-0@lo:8:kworker/u8:3.0 Request processed in 141us (938us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098880.959368:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1294
00000100:00100000:2.0:1766098880.959397:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893184
00000100:00100000:2.0:1766098880.959420:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2118
00000100:00100000:2.0:1766098880.959425:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2232645 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432893056:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098880.959554:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2232645 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432893056:12345-0@lo:400:kworker/u8:3.0 Request processed in 126us (1384us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098880.959563:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2119
00000100:00100000:2.0:1766098880.959581:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2118
00000100:00100000:0.0:1766098880.959582:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6c1f26a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432893184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098880.959642:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6c1f26a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432893184:12345-0@lo:400:kworker/u8:3.0 Request processed in 68us (1414us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098880.959651:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2119
00000100:00100000:2.0:1766098880.959658:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432893184:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098880.959675:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432893056:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098880.959718:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432892992
00000100:00100000:2.0:1766098880.959750:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1441
00000100:00100000:2.0:1766098880.959754:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432892992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098880.959790:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432892992:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (1413us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098880.959795:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1441
00000100:00100000:1.0:1766098880.962581:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c07d779d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432892992:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766098880.962611:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004dbc67d9 x1851885432893248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098887 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766098880.962625:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432893248:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766098880.962715:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098880.962719:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098880.962723:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432893120:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766098881.406348:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098882.429882:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098883.453729:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098884.478187:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098885.424016:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098885.424096:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098885.424156:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098885.424188:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197824
00000100:00100000:3.0:1766098885.424273:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1295
00000100:00100000:3.0:1766098885.424282:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008e55a8be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417197824:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098885.424386:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098885.424471:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008e55a8be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417197824:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 188us (460us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098885.424519:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1295
00000100:00100000:0.0:1766098885.424537:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197760
00000100:00100000:0.0:1766098885.424599:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2120
00000100:00100000:0.0:1766098885.424607:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417197760:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098885.424680:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417197760:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 74us (525us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098885.424688:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2120
00000100:00100000:1.0:1766098885.425637:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197696
00000100:00100000:1.0:1766098885.425709:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1442
00000100:00100000:1.0:1766098885.425715:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417197696:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098885.425781:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417197696:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 65us (1685us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098885.425789:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1442
00010000:00010000:2.0:1766098885.501701:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098886.079611:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432893312:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098886.079703:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766098886.080116:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098886.080121:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098886.080124:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098886.080128:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3389
00000100:00080000:1.0:1766098886.080131:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098886.080136:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098886.080153:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001a3c97aa x1851885432893440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098886.080202:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766098886.080208:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432893568:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098886.080237:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098886.080264:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432893376:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098886.080288:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098886.080306:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098886.080310:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432893504:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098886.080327:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098886.080357:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893376
00000100:00100000:2.0:1766098886.080390:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893504
00000100:00100000:2.0:1766098886.080407:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2121
00000100:00100000:2.0:1766098886.080411:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432893376:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098886.080450:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036f32338 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432893376:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (162us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098886.080456:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2121
00000100:00100000:2.0:1766098886.080469:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2122
00000100:00100000:2.0:1766098886.080473:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432893504:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098886.080516:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c07d779d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432893504:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (189us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098886.080522:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2122
00000100:00100000:0.0:1766098886.082053:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893312
00000100:00100000:0.0:1766098886.082091:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1443
00000100:00100000:0.0:1766098886.082095:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432893312:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098886.082157:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432893312:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (2455us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098886.082163:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1443
00000100:00100000:0.0:1766098886.082208:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432893440:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098886.082238:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098886.082292:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893568
00000100:00100000:0.0:1766098886.082313:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893440
00000100:00100000:0.0:1766098886.082328:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 985
00000100:00100000:0.0:1766098886.082332:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432893568:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098886.082363:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432893568:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (2125us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098886.082369:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 985
00000100:00100000:0.0:1766098886.082380:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1296
00000100:00100000:0.0:1766098886.082383:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432893440:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098886.082501:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098886.082532:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432893440:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (294us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098886.082538:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1296
00000100:00100000:0.0:1766098886.082580:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432893504:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098886.082592:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432893376:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766098886.082622:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098886.082638:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098886.082641:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432893440:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766098886.082851:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003a108df5 x1851885432893568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098893 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098886.082862:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432893568:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098886.082872:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432893312:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098886.526271:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098887.549847:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098888.573791:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098889.597733:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098890.527863:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098890.527918:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098890.528054:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098890.528166:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197888
00000100:00100000:3.0:1766098890.528204:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417197952
00000100:00100000:3.0:1766098890.528250:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2123
00000100:00100000:3.0:1766098890.528259:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417197952:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098890.528290:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198016
00000100:00100000:1.0:1766098890.528314:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1297
00000100:00100000:1.0:1766098890.528319:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417198016:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:3.0:1766098890.528362:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417197952:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 104us (444us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098890.528370:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2123
00010000:02020000:1.0:1766098890.528415:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098890.528447:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417198016:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 127us (394us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098890.528454:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1297
00000100:00100000:2.0:1766098890.528524:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1444
00000100:00100000:2.0:1766098890.528529:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417197888:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098890.528666:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417197888:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 135us (815us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098890.528673:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1444
00010000:00010000:0.0:1766098890.621766:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098891.198119:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098891.198130:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098891.198134:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098891.198144:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3394
00000100:00080000:3.0:1766098891.198148:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098891.198156:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098891.198181:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bb97adba x1851885432893760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098891.198202:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432893632:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198240:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432893696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098891.198367:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432893888:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198452:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098891.198511:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098891.198517:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432893824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198544:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098891.198575:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432893760:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198602:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098891.198643:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893696
00000100:00100000:3.0:1766098891.198691:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893824
00000100:00100000:3.0:1766098891.198715:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2124
00000100:00100000:3.0:1766098891.198721:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432893696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198772:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432893696:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (322us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098891.198781:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2124
00000100:00100000:3.0:1766098891.198817:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2125
00000100:00100000:3.0:1766098891.198820:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000632c907e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432893824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198844:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000632c907e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432893824:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (300us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098891.198849:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2125
00000100:00100000:3.0:1766098891.198895:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432893824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198906:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432893696:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.198957:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893760
00000100:00100000:3.0:1766098891.198996:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1298
00000100:00100000:3.0:1766098891.199000:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432893760:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766098891.199099:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098891.199140:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432893760:12345-0@lo:8:kworker/u8:3.0 Request processed in 138us (537us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098891.199146:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1298
00000100:00080000:3.0:1766098891.199180:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098891.199184:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098891.199187:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432893760:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098891.199375:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098891.199657:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098891.199733:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893888
00000100:00100000:3.0:1766098891.199769:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 986
00000100:00100000:3.0:1766098891.199773:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432893888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098891.199824:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432893888:12345-0@lo:400:kworker/u8:3.0 Request processed in 50us (168us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098891.199831:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 986
00010000:00080000:2.0:1766098891.200012:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e3400fcc x1851885432893888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098898 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766098891.200023:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432893888:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098891.200222:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893632
00000100:00100000:1.0:1766098891.201168:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1445
00000100:00100000:1.0:1766098891.201178:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432893632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098891.201270:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432893632:12345-0@lo:400:kworker/u8:3.0 Request processed in 92us (1897us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098891.201276:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1445
00000100:00100000:2.0:1766098891.201323:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432893632:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098891.645739:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098892.671233:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098893.695821:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098894.717695:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098895.644862:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098895.645775:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198208
00000100:00100000:3.0:1766098895.645863:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1299
00000100:00100000:3.0:1766098895.645872:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417198208:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098895.646001:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098895.646093:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417198208:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 221us (1243us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098895.646102:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1299
00000100:00100000:1.0:1766098895.648369:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098895.648427:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098895.648713:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198080
00000100:00100000:2.0:1766098895.648781:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1446
00000100:00100000:2.0:1766098895.648789:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417198080:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098895.649066:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198144
00000100:00100000:3.0:1766098895.649103:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2126
00000100:00100000:3.0:1766098895.649107:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417198144:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098895.649592:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417198080:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 801us (1231us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098895.649603:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1446
00000100:00100000:3.0:1766098895.649691:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba96c12 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417198144:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 582us (1264us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098895.649700:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2126
00010000:00010000:2.0:1766098895.743398:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098896.318661:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098896.318670:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098896.318674:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098896.318701:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3399
00000100:00080000:2.0:1766098896.318704:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098896.318710:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098896.318735:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008d577608 x1851885432894080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098896.318791:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432894016:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098896.318876:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098896.318914:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098896.318915:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bec75682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432894208:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098896.318927:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432894144:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098896.318972:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098896.318979:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098896.318994:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098896.319001:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432893952:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098896.319020:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432894080:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098896.319035:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098896.319051:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098896.319080:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894016
00000100:00100000:1.0:1766098896.319131:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894144
00000100:00100000:1.0:1766098896.319155:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2127
00000100:00100000:1.0:1766098896.319169:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432894016:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098896.319221:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432894016:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (347us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098896.319229:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2127
00000100:00100000:1.0:1766098896.319247:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2128
00000100:00100000:1.0:1766098896.319251:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653fe0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432894144:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098896.319276:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653fe0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432894144:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (306us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098896.319281:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2128
00000100:00100000:0.0:1766098896.319298:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894208
00000100:00100000:0.0:1766098896.319343:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894080
00000100:00100000:0.0:1766098896.319359:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 987
00000100:00100000:0.0:1766098896.319363:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432894208:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098896.319411:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432894208:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (439us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098896.319417:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 987
00000100:00100000:0.0:1766098896.319433:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1300
00000100:00100000:0.0:1766098896.319436:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432894080:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098896.319555:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098896.319584:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432894080:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (536us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098896.319590:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1300
00010000:00080000:1.0:1766098896.319710:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bec75682 x1851885432894208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098903 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098896.319723:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bec75682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432894208:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766098896.319723:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098896.319730:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098896.319735:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432894080:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098896.319766:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432894144:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098896.319779:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432894016:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098896.319808:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432893952
00000100:00100000:2.0:1766098896.320423:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1447
00000100:00100000:2.0:1766098896.320430:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432893952:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098896.320531:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432893952:12345-0@lo:400:kworker/u8:0.0 Request processed in 101us (1496us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098896.320539:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1447
00000100:00100000:0.0:1766098896.320671:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432893952:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098896.766315:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098897.790334:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098898.813835:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098899.839179:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098900.765748:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098900.767311:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198272
00000100:00100000:1.0:1766098900.767385:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1448
00000100:00100000:1.0:1766098900.767392:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417198272:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098900.770240:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417198272:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2839us (4491us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098900.770257:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1448
00000100:00100000:2.0:1766098900.771146:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098900.771189:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098900.771518:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198336
00000100:00100000:3.0:1766098900.771576:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2129
00000100:00100000:3.0:1766098900.771582:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a2ebda9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417198336:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098900.771649:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a2ebda9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417198336:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 66us (506us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098900.771655:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2129
00000100:00100000:1.0:1766098900.771855:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198400
00000100:00100000:1.0:1766098900.771886:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1301
00000100:00100000:1.0:1766098900.771891:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417198400:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098900.772020:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098900.772049:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417198400:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 158us (861us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098900.772056:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1301
00010000:00010000:2.0:1766098900.862029:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098901.437861:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098901.437869:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098901.437873:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098901.437880:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3404
00000100:00080000:2.0:1766098901.437884:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098901.437889:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098901.437940:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d4e5a5c3 x1851885432894400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098901.438107:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bec75682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432894464:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098901.438173:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098901.438206:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098901.438211:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039d784a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432894336:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098901.438232:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098901.438259:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894464
00000100:00100000:2.0:1766098901.438290:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894336
00000100:00100000:2.0:1766098901.438308:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2130
00000100:00100000:2.0:1766098901.438313:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7b9f638 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432894464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098901.438356:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7b9f638 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432894464:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (183us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098901.438362:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2130
00000100:00100000:2.0:1766098901.438377:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2131
00000100:00100000:2.0:1766098901.438380:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432894336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098901.438401:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432894336:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (170us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098901.438406:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2131
00000100:00100000:2.0:1766098901.438441:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432894400:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098901.438474:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098901.438533:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894400
00000100:00100000:1.0:1766098901.438548:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432894528:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098901.438556:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1302
00000100:00100000:2.0:1766098901.438560:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432894400:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098901.438581:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098901.438602:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098901.438608:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432894272:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098901.438633:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:02020000:2.0:1766098901.438649:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098901.438661:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894528
00000100:00100000:2.0:1766098901.438684:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432894400:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (210us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098901.438691:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1302
00000100:00080000:3.0:1766098901.439094:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098901.439099:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098901.439103:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432894400:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098901.439132:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039d784a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432894336:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098901.439141:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bec75682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432894464:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098901.439189:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 988
00000100:00100000:1.0:1766098901.439195:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432894528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098901.439234:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432894528:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (654us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098901.439240:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 988
00010000:00080000:1.0:1766098901.439271:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000369e20f7 x1851885432894528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098908 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098901.439279:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432894528:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098901.439297:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894272
00000100:00100000:1.0:1766098901.440364:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1449
00000100:00100000:1.0:1766098901.440369:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432894272:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098901.440421:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432894272:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (1787us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098901.440427:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1449
00000100:00100000:1.0:1766098901.440496:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069da214f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432894272:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098901.886585:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098902.909655:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098903.933696:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098904.957818:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098905.890197:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098905.890255:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098905.890352:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198528
00000100:00100000:3.0:1766098905.890446:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2132
00000100:00100000:3.0:1766098905.890455:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417198528:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098905.890610:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417198528:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 155us (355us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098905.890619:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2132
00000100:00100000:1.0:1766098905.893580:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198464
00000100:00100000:2.0:1766098905.894280:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1450
00000100:00100000:2.0:1766098905.894289:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417198464:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098905.894383:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417198464:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 94us (4193us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098905.894391:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1450
00000100:00100000:2.0:1766098905.900564:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098905.901779:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198592
00000100:00100000:0.0:1766098905.901844:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1303
00000100:00100000:0.0:1766098905.901850:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417198592:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098905.901951:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098905.902020:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417198592:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 169us (1464us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098905.902027:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1303
00010000:00010000:0.0:1766098905.981710:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098906.557628:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098906.557636:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098906.557639:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098906.557647:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3409
00000100:00080000:2.0:1766098906.557650:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098906.557655:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098906.557686:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c97d50d6 x1851885432894720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098906.557765:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034f090f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432894656:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098906.557826:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098906.557872:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098906.557878:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007aef029b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432894784:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098906.557897:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098906.557924:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432894592:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098906.557926:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432894720:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098906.557961:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098906.557978:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098906.558001:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894656
00000100:00100000:0.0:1766098906.558004:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766098906.558011:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432894848:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098906.558037:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098906.558045:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894784
00000100:00100000:3.0:1766098906.558073:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2133
00000100:00100000:0.0:1766098906.558075:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894720
00000100:00100000:3.0:1766098906.558080:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432894656:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098906.558100:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894848
00000100:00100000:0.0:1766098906.558125:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 989
00000100:00100000:0.0:1766098906.558129:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432894848:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098906.558135:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432894656:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (306us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098906.558142:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2133
00000100:00100000:3.0:1766098906.558161:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2134
00000100:00100000:3.0:1766098906.558165:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bec75682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432894784:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098906.558177:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432894848:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (139us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098906.558184:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 989
00000100:00100000:0.0:1766098906.558202:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1304
00000100:00100000:3.0:1766098906.558207:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bec75682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432894784:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (309us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098906.558208:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d784a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432894720:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098906.558230:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2134
00000100:00100000:1.0:1766098906.558257:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894592
00000100:00100000:3.0:1766098906.558271:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007aef029b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432894784:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098906.558281:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034f090f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432894656:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098906.558290:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1451
00000100:00100000:1.0:1766098906.558293:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432894592:12345-0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766098906.558300:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098906.558331:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432894592:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (354us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098906.558339:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1451
00000100:00100000:0.0:1766098906.558347:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d784a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432894720:12345-0@lo:8:kworker/u8:0.0 Request processed in 138us (383us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098906.558354:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1304
00010000:00080000:1.0:1766098906.558386:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000065bd5068 x1851885432894848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098913 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098906.558394:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432894848:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098906.558401:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432894592:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098906.558550:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098906.558554:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098906.558558:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432894720:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098907.005742:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098908.031068:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098909.055506:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098910.079434:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098911.004748:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098911.005187:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198784
00000100:00100000:2.0:1766098911.005259:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1305
00000100:00100000:2.0:1766098911.005271:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417198784:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098911.005377:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098911.005468:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417198784:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 200us (731us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098911.005513:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1305
00000100:00100000:2.0:1766098911.010212:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098911.010359:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198656
00000100:00100000:1.0:1766098911.010431:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1452
00000100:00100000:1.0:1766098911.010439:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417198656:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098911.010571:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417198656:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 131us (371us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098911.010579:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1452
00000100:00100000:0.0:1766098911.012296:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098911.012384:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198720
00000100:00100000:2.0:1766098911.012453:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2135
00000100:00100000:2.0:1766098911.012460:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004ba22e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417198720:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098911.012584:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004ba22e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417198720:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 124us (296us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098911.012592:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2135
00010000:00010000:2.0:1766098911.101761:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098911.677739:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766098911.677745:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098911.677753:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:3.0:1766098911.677761:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432894912:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098911.677864:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098911.677906:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432894976:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098911.677937:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098911.677986:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894912
00000100:00100000:3.0:1766098911.678041:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1453
00000100:00100000:3.0:1766098911.678049:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432894912:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098911.678099:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432894912:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (237us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098911.678107:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1453
00000100:00100000:3.0:1766098911.678147:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432894976
00000100:00100000:3.0:1766098911.678177:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2136
00000100:00100000:3.0:1766098911.678180:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432894976:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098911.678212:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432894976:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (276us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098911.678218:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2136
00000100:00100000:3.0:1766098911.678256:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432894976:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098911.678279:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432894912:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766098911.678731:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098911.678745:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3415
00000100:00080000:2.0:1766098911.678749:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098911.678757:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098911.678805:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000065bd5068 x1851885432895040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098911.679017:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432895040:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098911.679077:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098911.679084:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098911.679094:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007aef029b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432895168:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098911.679148:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098911.679232:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895040
00000100:00100000:2.0:1766098911.679262:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895168
00000100:00100000:2.0:1766098911.679285:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 990
00000100:00100000:2.0:1766098911.679291:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432895168:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098911.679344:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432895168:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (199us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098911.679351:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 990
00000100:00100000:2.0:1766098911.679366:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1306
00000100:00100000:2.0:1766098911.679369:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047fe747c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432895040:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098911.679564:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098911.679575:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034f090f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432895104:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098911.679607:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:0.0:1766098911.679993:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007aef029b x1851885432895168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098918 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766098911.680007:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007aef029b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432895168:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098911.680056:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895104
00010000:02020000:2.0:1766098911.680292:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098911.680340:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047fe747c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432895040:12345-0@lo:8:kworker/u8:3.0 Request processed in 968us (1258us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098911.680349:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1306
00000100:00080000:2.0:1766098911.680387:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098911.680390:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098911.680394:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432895040:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098911.680821:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2137
00000100:00100000:3.0:1766098911.680857:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432895104:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098911.680948:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432895104:12345-0@lo:400:kworker/u8:3.0 Request processed in 92us (1340us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098911.680956:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2137
00000100:00100000:1.0:1766098911.681249:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034f090f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432895104:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098912.126658:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098913.150244:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098914.177181:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098915.198662:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098916.124826:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098916.125147:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098916.125187:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098916.125274:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198912
00000100:00100000:1.0:1766098916.125379:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2138
00000100:00100000:1.0:1766098916.125391:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7b70f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417198912:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098916.125407:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198848
00000100:00100000:2.0:1766098916.125456:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1454
00000100:00100000:2.0:1766098916.125465:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417198848:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098916.125554:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7b70f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417198912:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 165us (407us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098916.125570:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417198848:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 95us (371us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098916.125583:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2138
00000100:00100000:2.0:1766098916.125608:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1454
00000100:00100000:3.0:1766098916.125780:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417198976
00000100:00100000:3.0:1766098916.125825:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1307
00000100:00100000:3.0:1766098916.125829:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417198976:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098916.125918:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098916.125964:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417198976:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 134us (1173us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098916.125971:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1307
00010000:00010000:2.0:1766098916.222261:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098916.799519:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098916.799528:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098916.799533:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098916.799543:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3420
00000100:00080000:0.0:1766098916.799547:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098916.799554:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098916.799580:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a5105a15 x1851885432895360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098916.799633:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432895360:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098916.799883:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432895488:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.799925:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432895296:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.799967:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098916.800001:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766098916.800007:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432895424:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.800031:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098916.800073:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895296
00000100:00100000:1.0:1766098916.800124:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895424
00000100:00100000:1.0:1766098916.800161:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2139
00000100:00100000:1.0:1766098916.800168:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432895296:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.800223:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432895296:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (257us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098916.800232:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2139
00000100:00100000:1.0:1766098916.800252:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2140
00000100:00100000:1.0:1766098916.800255:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c91368f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432895424:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.800301:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c91368f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432895424:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (269us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098916.800308:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2140
00000100:00100000:1.0:1766098916.800348:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432895424:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.800359:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432895296:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098916.800867:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098916.800921:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895360
00000100:00100000:0.0:1766098916.800935:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098916.801138:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432895232:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098916.801192:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098916.801231:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895232
00000100:00100000:2.0:1766098916.801262:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1455
00000100:00100000:2.0:1766098916.801266:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432895232:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098916.801304:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432895232:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (112us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098916.801311:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1455
00000100:00100000:2.0:1766098916.801350:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432895232:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.802321:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895488
00000100:00100000:3.0:1766098916.802334:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1308
00000100:00100000:3.0:1766098916.802342:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432895360:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098916.802378:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 991
00000100:00100000:1.0:1766098916.802383:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432895488:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098916.802437:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432895488:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (1502us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098916.802444:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 991
00010000:02020000:3.0:1766098916.802451:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098916.802526:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432895360:12345-0@lo:8:kworker/u8:0.0 Request processed in 182us (1659us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098916.802534:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1308
00000100:00080000:1.0:1766098916.802558:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098916.802563:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098916.802569:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432895360:0@lo:8:kworker/u8:0.0
00010000:00080000:0.0:1766098916.802762:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c63c9d57 x1851885432895488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098923 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098916.802799:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432895488:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098917.248144:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098918.271449:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098919.293722:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098920.317777:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098921.251555:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098921.252969:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098921.254633:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199104
00000100:00100000:3.0:1766098921.254728:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2141
00000100:00100000:3.0:1766098921.254737:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417199104:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098921.255015:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199040
00000100:00100000:2.0:1766098921.255048:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1456
00000100:00100000:2.0:1766098921.255066:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417199040:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098921.255114:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417199104:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 377us (2149us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098921.255122:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2141
00000100:00100000:2.0:1766098921.255702:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417199040:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 646us (4158us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098921.255710:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1456
00000100:00100000:2.0:1766098921.255989:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098921.256459:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199168
00000100:00100000:3.0:1766098921.256531:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1309
00000100:00100000:3.0:1766098921.256536:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417199168:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098921.256644:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098921.256713:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417199168:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 176us (726us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098921.256720:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1309
00010000:00010000:0.0:1766098921.342641:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098921.917921:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766098921.918447:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098921.918453:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098921.918457:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098921.918468:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3425
00000100:00080000:3.0:1766098921.918473:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098921.918502:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098921.918529:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004dbc67d9 x1851885432895680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098921.919613:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432895744:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098921.919693:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098921.919728:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098921.919761:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432895616:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098921.919791:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098921.919806:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432895808:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098921.919845:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895744
00000100:00100000:1.0:1766098921.919849:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098921.919871:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432895552:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098921.919897:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895616
00000100:00100000:1.0:1766098921.919907:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098921.919924:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2142
00000100:00100000:2.0:1766098921.919931:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034f090f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432895744:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098921.919951:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432895680:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098921.919988:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098921.919997:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034f090f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432895744:12345-0@lo:400:kworker/u8:0.0 Request processed in 65us (304us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098921.920007:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2142
00000100:00100000:1.0:1766098921.920027:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2143
00000100:00100000:1.0:1766098921.920034:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432895616:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098921.920049:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895552
00000100:00100000:1.0:1766098921.920080:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432895616:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (287us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098921.920084:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1457
00000100:00100000:1.0:1766098921.920092:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2143
00000100:00100000:2.0:1766098921.920093:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432895552:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098921.920146:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432895552:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (239us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098921.920153:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1457
00000100:00100000:1.0:1766098921.920159:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895808
00000100:00100000:1.0:1766098921.920196:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895680
00000100:00100000:2.0:1766098921.920206:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb97adba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432895616:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098921.920220:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 992
00000100:00100000:2.0:1766098921.920223:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432895744:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098921.920229:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432895808:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098921.920266:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432895808:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (417us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098921.920272:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1310
00000100:00100000:1.0:1766098921.920279:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 992
00000100:00100000:2.0:1766098921.920280:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432895680:12345-0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766098921.920317:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e27d7f39 x1851885432895808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098928 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098921.920327:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432895808:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098921.920335:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005114f03e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432895552:0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766098921.920373:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098921.920419:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432895680:12345-0@lo:8:kworker/u8:0.0 Request processed in 141us (431us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098921.920425:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1310
00000100:00080000:1.0:1766098921.920563:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098921.920568:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098921.920571:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432895680:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098922.365804:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098923.389908:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098924.413816:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098925.438344:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098926.366876:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098926.370547:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199232
00000100:00100000:0.0:1766098926.370671:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1458
00000100:00100000:0.0:1766098926.370680:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417199232:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098926.371385:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417199232:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 703us (4520us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098926.371397:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1458
00000100:00100000:1.0:1766098926.371733:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098926.371779:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098926.373086:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199296
00000100:00100000:3.0:1766098926.373345:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199360
00000100:00100000:3.0:1766098926.373380:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1311
00000100:00100000:3.0:1766098926.373387:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417199360:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098926.373506:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098926.373585:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417199360:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 198us (1806us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098926.373593:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1311
00000100:00100000:0.0:1766098926.373819:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2144
00000100:00100000:0.0:1766098926.373825:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d77d75e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417199296:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098926.373892:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d77d75e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417199296:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 66us (2160us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098926.373899:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2144
00010000:00010000:2.0:1766098926.463187:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098927.037872:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098927.037881:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098927.037885:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098927.037895:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3430
00000100:00080000:2.0:1766098927.037899:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098927.037906:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098927.037933:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004dbc67d9 x1851885432896000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098927.038052:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432895872:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098927.038140:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098927.038434:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098927.038444:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432895936:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.038510:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098927.038542:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896000:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098927.038567:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098927.038619:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098927.038625:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432896064:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.038646:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098927.038676:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895936
00000100:00100000:2.0:1766098927.038859:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896064
00000100:00100000:2.0:1766098927.038884:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2145
00000100:00100000:2.0:1766098927.038891:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432895936:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.038998:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005114f03e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432895936:12345-0@lo:400:kworker/u8:3.0 Request processed in 107us (487us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098927.039006:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2145
00000100:00100000:2.0:1766098927.039028:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2146
00000100:00100000:2.0:1766098927.039032:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432896064:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.039078:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432896064:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (431us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098927.039084:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2146
00000100:00100000:2.0:1766098927.039123:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432896064:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.039178:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432895936:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.039195:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432896128:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098927.039217:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098927.039314:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432895872
00000100:00100000:3.0:1766098927.039357:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1459
00000100:00100000:3.0:1766098927.039361:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432895872:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098927.039407:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432895872:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (1268us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098927.039414:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1459
00000100:00100000:2.0:1766098927.039439:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896000
00000100:00100000:2.0:1766098927.040024:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896128
00000100:00100000:0.0:1766098927.040060:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 993
00000100:00100000:0.0:1766098927.040066:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432896128:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098927.040114:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432896128:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (897us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098927.040120:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 993
00000100:00100000:0.0:1766098927.040132:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1312
00000100:00100000:0.0:1766098927.040142:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb97adba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432896000:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098927.040233:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098927.040265:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb97adba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432896000:12345-0@lo:8:kworker/u8:3.0 Request processed in 123us (1698us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098927.040271:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1312
00000100:00080000:0.0:1766098927.040300:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098927.040303:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098927.040307:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dbc67d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896000:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766098927.040332:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e6e5460e x1851885432896128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098934 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766098927.040341:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432896128:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098927.040366:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432895872:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098927.485874:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098928.509859:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098929.533921:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098930.559415:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098931.487835:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098931.489356:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098931.489398:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098931.489532:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199424
00000100:00100000:0.0:1766098931.489550:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199488
00000100:00100000:0.0:1766098931.489643:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2147
00000100:00100000:0.0:1766098931.489650:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002684e04b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417199488:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098931.489724:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199552
00000100:00100000:0.0:1766098931.489750:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002684e04b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417199488:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 100us (1923us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098931.489758:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2147
00000100:00100000:1.0:1766098931.490643:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1313
00000100:00100000:1.0:1766098931.490673:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417199552:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766098931.490787:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098931.490865:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417199552:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 193us (1466us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098931.490872:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1313
00000100:00100000:3.0:1766098931.496578:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1460
00000100:00100000:3.0:1766098931.496586:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417199424:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098931.496665:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417199424:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 79us (7314us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098931.496672:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1460
00010000:00010000:2.0:1766098931.581740:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098932.158227:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098932.158234:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098932.158238:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098932.158246:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3435
00000100:00080000:2.0:1766098932.158250:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098932.158255:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098932.158276:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002569b802 x1851885432896320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098932.158362:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432896448:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098932.158423:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098932.158450:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766098932.158456:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432896192:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098932.158522:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098932.158660:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896320:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098932.158700:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098932.158734:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896448
00000100:00100000:1.0:1766098932.158782:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896320
00000100:00100000:1.0:1766098932.158802:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 994
00000100:00100000:1.0:1766098932.158808:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432896448:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098932.158848:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432896448:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (426us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098932.158855:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 994
00000100:00100000:1.0:1766098932.158871:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1314
00000100:00100000:1.0:1766098932.158875:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432896320:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098932.158966:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098932.158996:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432896320:12345-0@lo:8:kworker/u8:0.0 Request processed in 120us (297us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098932.159003:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1314
00000100:00100000:2.0:1766098932.159042:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432896384:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098932.159043:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098932.159049:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098932.159054:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002569b802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896320:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098932.159102:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098932.159124:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098932.159131:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432896256:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098932.159135:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896384
00000100:00100000:2.0:1766098932.159155:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:0.0:1766098932.159451:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000aea833b9 x1851885432896448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098939 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098932.159461:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432896448:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098932.159554:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896256
00000100:00100000:1.0:1766098932.159564:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2148
00000100:00100000:1.0:1766098932.159570:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432896384:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098932.159583:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2149
00000100:00100000:2.0:1766098932.159589:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8700567 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432896256:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098932.159614:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432896384:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (512us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098932.159623:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2148
00000100:00100000:2.0:1766098932.159634:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8700567 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432896256:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (478us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098932.159642:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2149
00000100:00100000:1.0:1766098932.159660:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896192
00000100:00100000:2.0:1766098932.159688:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432896256:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098932.159701:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dd0a2da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432896384:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098932.159707:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1461
00000100:00100000:1.0:1766098932.159713:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432896192:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098932.159751:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432896192:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (1229us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098932.159770:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432896192:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098932.159942:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1461
00010000:00010000:2.0:1766098932.606922:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098933.629833:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098934.656817:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098935.677748:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098936.607853:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098936.607901:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098936.607939:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199680
00000100:00100000:0.0:1766098936.607982:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098936.608009:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2150
00000100:00100000:0.0:1766098936.608011:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199744
00000100:00100000:2.0:1766098936.608018:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417199680:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098936.608035:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1315
00000100:00100000:0.0:1766098936.608041:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417199744:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766098936.608097:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417199680:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 79us (250us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098936.608105:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2150
00010000:02020000:0.0:1766098936.608145:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098936.608175:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417199744:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 133us (193us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098936.608181:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1315
00000100:00100000:1.0:1766098936.611954:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199616
00000100:00100000:1.0:1766098936.612020:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1462
00000100:00100000:1.0:1766098936.612026:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417199616:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098936.612103:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417199616:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (4202us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098936.612110:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1462
00010000:00010000:2.0:1766098936.701650:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098937.277703:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098937.277728:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098937.277732:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098937.277739:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3440
00000100:00080000:2.0:1766098937.277742:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098937.277748:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098937.277769:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b7ebad2d x1851885432896640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098937.277820:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432896704:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098937.277856:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bfc1948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432896512:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.277879:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098937.277918:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098937.277927:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098937.277935:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432896576:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098937.277947:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098937.277956:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432896768:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.277965:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098937.277994:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098937.278012:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896704
00000100:00100000:2.0:1766098937.278049:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896576
00000100:00100000:2.0:1766098937.278068:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2151
00000100:00100000:2.0:1766098937.278073:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432896704:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098937.278094:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7ebad2d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896640:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278126:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432896704:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (247us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098937.278134:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098937.278138:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2151
00000100:00100000:2.0:1766098937.278156:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2152
00000100:00100000:2.0:1766098937.278161:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432896576:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278188:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432896576:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (224us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098937.278194:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2152
00000100:00100000:2.0:1766098937.278230:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432896576:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278252:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432896704:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278293:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896768
00000100:00100000:2.0:1766098937.278317:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896640
00000100:00100000:2.0:1766098937.278334:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 995
00000100:00100000:2.0:1766098937.278338:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432896768:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278377:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432896768:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (387us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098937.278385:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 995
00000100:00100000:2.0:1766098937.278401:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1316
00000100:00100000:2.0:1766098937.278404:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432896640:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098937.278517:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098937.278551:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432896640:12345-0@lo:8:kworker/u8:0.0 Request processed in 145us (421us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098937.278558:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1316
00000100:00080000:2.0:1766098937.278587:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098937.278591:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098937.278595:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7ebad2d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896640:0@lo:8:kworker/u8:0.0
00010000:00080000:2.0:1766098937.278618:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e6e5460e x1851885432896768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098944 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098937.278627:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432896768:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278674:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896512
00000100:00100000:2.0:1766098937.278701:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1463
00000100:00100000:2.0:1766098937.278705:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432896512:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098937.278737:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432896512:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (821us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098937.278749:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1463
00000100:00100000:2.0:1766098937.278782:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bfc1948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432896512:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098937.725727:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098938.749735:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098939.774195:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098940.797910:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098941.728217:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098941.728261:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098941.728292:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098941.728368:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199936
00000100:00100000:3.0:1766098941.728410:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1317
00000100:00100000:3.0:1766098941.728416:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417199936:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766098941.728554:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098941.728618:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417199936:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 202us (326us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098941.728625:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1317
00000100:00100000:1.0:1766098941.729040:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199872
00000100:00100000:1.0:1766098941.729098:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2153
00000100:00100000:1.0:1766098941.729103:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004722f0a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417199872:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098941.729152:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004722f0a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417199872:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 48us (890us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098941.729159:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2153
00000100:00100000:2.0:1766098941.729670:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417199808
00000100:00100000:2.0:1766098941.729705:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1464
00000100:00100000:2.0:1766098941.729710:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417199808:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766098941.729751:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417199808:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 40us (1541us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098941.729758:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1464
00010000:00010000:0.0:1766098941.821760:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098942.398462:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766098942.398494:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1766098942.398495:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432896832:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098942.398500:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098942.398503:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098942.398508:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3445
00000100:00080000:3.0:1766098942.398510:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098942.398515:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766098942.398520:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098942.398532:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432896896:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098942.398541:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004039bd81 x1851885432896960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098942.398558:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098942.398614:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098942.398622:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098942.398625:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432897024:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098942.398633:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432897088:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098942.398676:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098942.398706:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896960:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098942.398731:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098942.398761:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896960
00000100:00100000:2.0:1766098942.398804:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1318
00000100:00100000:2.0:1766098942.398810:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432896960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766098942.398918:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098942.398952:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432896960:12345-0@lo:8:kworker/u8:0.0 Request processed in 142us (221us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098942.398959:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1318
00000100:00080000:2.0:1766098942.398996:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098942.398999:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098942.399003:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432896960:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766098942.399030:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896896
00000100:00100000:2.0:1766098942.399055:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2154
00000100:00100000:2.0:1766098942.399059:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432896896:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098942.399095:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432896896:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (424us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098942.399101:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2154
00000100:00100000:2.0:1766098942.399124:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432896896:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098942.399212:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432896832
00000100:00100000:2.0:1766098942.399236:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1465
00000100:00100000:2.0:1766098942.399239:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432896832:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098942.399265:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432896832:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (709us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098942.399271:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1465
00000100:00100000:0.0:1766098942.399816:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432896832:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098942.399892:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098942.399986:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897024
00000100:00100000:0.0:1766098942.400019:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2155
00000100:00100000:0.0:1766098942.400023:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002569b802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432897024:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098942.401782:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098942.401846:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897088
00000100:00100000:3.0:1766098942.401881:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 996
00000100:00100000:3.0:1766098942.401886:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432897088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098942.401951:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432897088:12345-0@lo:400:kworker/u8:0.0 Request processed in 64us (174us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098942.401960:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 996
00010000:00080000:2.0:1766098942.401991:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000066defd5f x1851885432897088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098949 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766098942.402004:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432897088:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098942.402159:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002569b802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432897024:12345-0@lo:400:kworker/u8:0.0 Request processed in 2135us (2268us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098942.402167:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2155
00000100:00100000:1.0:1766098942.402716:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432897024:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098942.845815:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098943.869688:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098944.893749:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098945.917735:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098946.847607:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098946.847778:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200000
00000100:00100000:3.0:1766098946.847790:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098946.847832:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1466
00000100:00100000:3.0:1766098946.847930:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098946.847958:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200128
00000100:00100000:0.0:1766098946.847961:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417200000:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098946.847987:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1319
00000100:00100000:3.0:1766098946.847991:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417200128:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766098946.848028:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417200000:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 69us (428us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098946.848036:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1466
00000100:00100000:1.0:1766098946.848559:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200064
00000100:00100000:1.0:1766098946.848643:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2156
00000100:00100000:1.0:1766098946.848647:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009bbfcf2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417200064:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098946.848697:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009bbfcf2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417200064:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 48us (907us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098946.848704:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2156
00010000:02020000:3.0:1766098946.848865:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098946.848894:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417200128:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 902us (966us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098946.848901:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1319
00010000:00010000:2.0:1766098946.942270:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098947.518012:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098947.518019:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098947.518022:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098947.518031:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3450
00000100:00080000:0.0:1766098947.518034:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098947.518039:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098947.518058:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000039dc5372 x1851885432897280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098947.518148:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432897344:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.518214:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098947.518268:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098947.518274:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432897216:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.518299:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098947.518325:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432897280:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098947.518351:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098947.518387:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897344
00000100:00100000:2.0:1766098947.518424:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897216
00000100:00100000:2.0:1766098947.518445:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2157
00000100:00100000:2.0:1766098947.518451:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7ebad2d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432897344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.518534:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7ebad2d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432897344:12345-0@lo:400:kworker/u8:3.0 Request processed in 83us (321us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098947.518542:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2157
00000100:00100000:2.0:1766098947.518571:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2158
00000100:00100000:2.0:1766098947.518575:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aea833b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432897216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.518601:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aea833b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432897216:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (303us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098947.518607:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2158
00000100:00100000:2.0:1766098947.518969:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432897216:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.518981:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432897344:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.519053:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897280
00000100:00100000:2.0:1766098947.519074:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1320
00000100:00100000:2.0:1766098947.519078:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e88137a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432897280:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766098947.519169:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098947.519204:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e88137a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432897280:12345-0@lo:8:kworker/u8:3.0 Request processed in 125us (852us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098947.519219:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1320
00000100:00080000:2.0:1766098947.519353:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766098947.519357:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766098947.519361:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432897280:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098947.520526:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432897408:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098947.520592:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098947.520616:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766098947.520622:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432897152:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098947.520650:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098947.520686:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897152
00000100:00100000:1.0:1766098947.520722:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1467
00000100:00100000:1.0:1766098947.520727:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432897152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098947.520771:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432897152:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (120us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098947.520790:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1467
00000100:00100000:1.0:1766098947.520839:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432897152:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.522740:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897408
00000100:00100000:2.0:1766098947.522803:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 997
00000100:00100000:2.0:1766098947.522809:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432897408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098947.523659:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432897408:12345-0@lo:400:kworker/u8:3.0 Request processed in 846us (3064us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098947.523671:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 997
00010000:00080000:3.0:1766098947.523869:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e26029d4 x1851885432897408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098954 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098947.523885:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e26029d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432897408:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098947.968577:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098948.990263:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098950.013684:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098951.040440:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098951.965891:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098951.967612:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098951.967730:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098951.969513:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200192
00000100:00100000:1.0:1766098951.969602:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1468
00000100:00100000:1.0:1766098951.969610:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417200192:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098951.969864:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417200192:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 244us (3977us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098951.969881:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1468
00000100:00100000:0.0:1766098951.970643:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200256
00000100:00100000:0.0:1766098951.970713:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2159
00000100:00100000:0.0:1766098951.970719:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417200256:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098951.970780:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417200256:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 60us (3050us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098951.972792:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2159
00000100:00100000:2.0:1766098951.972884:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200320
00000100:00100000:2.0:1766098951.972967:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1321
00000100:00100000:2.0:1766098951.972975:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417200320:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766098951.973073:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098951.973160:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417200320:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 184us (5557us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098951.973168:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1321
00010000:00010000:2.0:1766098952.061854:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766098952.638329:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098952.638336:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098952.638340:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098952.638348:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3455
00000100:00080000:3.0:1766098952.638351:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098952.638356:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098952.638376:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000066defd5f x1851885432897600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098952.638440:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432897664:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098952.638459:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432897600:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098952.638718:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092cf9228 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432897472:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098952.638821:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098952.638876:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897600
00000100:00100000:3.0:1766098952.638912:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1322
00000100:00100000:3.0:1766098952.638920:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432897600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098952.639012:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098952.639051:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432897600:12345-0@lo:8:kworker/u8:0.0 Request processed in 131us (232us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098952.639059:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1322
00000100:00080000:3.0:1766098952.639094:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098952.639098:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098952.639101:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432897600:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098952.639134:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432897728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098952.639157:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098952.639180:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897728
00000100:00100000:3.0:1766098952.639204:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 998
00000100:00100000:3.0:1766098952.639220:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432897728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098952.639261:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432897728:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (104us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098952.639266:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 998
00010000:00080000:3.0:1766098952.639289:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fcd4deb0 x1851885432897728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098959 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098952.639298:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432897728:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098952.639688:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098952.639714:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766098952.639721:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432897536:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098952.639756:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098952.639784:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897664
00000100:00100000:2.0:1766098952.639808:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897536
00000100:00100000:2.0:1766098952.639824:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2160
00000100:00100000:2.0:1766098952.639828:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432897664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098952.639869:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432897664:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (181us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098952.639876:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2160
00000100:00100000:2.0:1766098952.639891:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2161
00000100:00100000:2.0:1766098952.639895:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000362df469 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432897536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098952.639920:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000362df469 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432897536:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (165us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098952.639926:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2161
00000100:00100000:2.0:1766098952.640345:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432897536:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098952.640360:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432897664:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098952.641292:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098952.641380:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897472
00000100:00100000:1.0:1766098952.641425:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1469
00000100:00100000:1.0:1766098952.641430:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432897472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098952.641507:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432897472:12345-0@lo:400:kworker/u8:0.0 Request processed in 76us (217us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098952.641514:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1469
00000100:00100000:1.0:1766098952.641578:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092cf9228 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432897472:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098953.085700:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098954.109684:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098955.134191:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098956.161735:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098957.088052:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098957.088834:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098957.089885:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098957.090041:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200512
00000100:00100000:3.0:1766098957.090387:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200448
00000100:00100000:3.0:1766098957.090448:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2162
00000100:00100000:3.0:1766098957.090456:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417200448:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.090587:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417200448:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 131us (1756us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098957.090597:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2162
00000100:00100000:2.0:1766098957.090688:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200384
00000100:00100000:2.0:1766098957.090748:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1470
00000100:00100000:2.0:1766098957.090753:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417200384:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098957.090810:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417200384:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 56us (2771us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098957.090817:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1470
00000100:00100000:1.0:1766098957.090874:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1323
00000100:00100000:1.0:1766098957.090880:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417200512:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766098957.090982:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098957.091017:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417200512:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 137us (1138us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098957.091025:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1323
00010000:00010000:0.0:1766098957.186327:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098957.757854:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098957.757863:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098957.757866:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098957.757876:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3461
00000100:00080000:0.0:1766098957.757880:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098957.757887:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098957.757915:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e4dd01a1 x1851885432897920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098957.757968:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432897920:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098957.758047:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098957.758106:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432897856:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098957.758136:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098957.758152:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098957.758158:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432897984:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098957.758179:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098957.758299:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432898048:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.758333:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098957.758363:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098957.758368:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432897792:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.758398:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098957.758436:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897792
00000100:00100000:3.0:1766098957.758520:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1471
00000100:00100000:3.0:1766098957.758526:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432897792:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.758580:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432897792:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (181us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098957.758588:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1471
00000100:00100000:3.0:1766098957.758663:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432897792:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.758690:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897856
00000100:00100000:3.0:1766098957.758716:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897984
00000100:00100000:1.0:1766098957.759273:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432897920
00000100:00100000:1.0:1766098957.759308:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898048
00000100:00100000:1.0:1766098957.759334:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 999
00000100:00100000:1.0:1766098957.759339:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432898048:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.759352:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2163
00000100:00100000:1.0:1766098957.759395:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432898048:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (1061us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098957.759402:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 999
00000100:00100000:0.0:1766098957.759409:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1324
00000100:00100000:0.0:1766098957.759416:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432897920:12345-0@lo:8:kworker/u8:3.0
00010000:00080000:1.0:1766098957.759447:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004c0ecbea x1851885432898048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098964 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766098957.759460:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432898048:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098957.759551:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2164
00000100:00100000:1.0:1766098957.759556:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432897984:12345-0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766098957.759565:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098957.759612:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432897920:12345-0@lo:8:kworker/u8:3.0 Request processed in 196us (1566us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098957.759619:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432897984:12345-0@lo:400:kworker/u8:3.0 Request processed in 61us (1439us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098957.759622:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1324
00000100:00100000:1.0:1766098957.759643:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2164
00000100:00080000:0.0:1766098957.759655:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098957.759660:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098957.759666:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432897920:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766098957.759684:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432897984:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098957.759791:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432897856:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098957.760237:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432897856:12345-0@lo:400:kworker/u8:3.0 Request processed in 446us (2099us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098957.760252:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2163
00000100:00100000:1.0:1766098957.761135:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432897856:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098958.209128:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098959.229688:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098960.256111:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098961.279255:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098962.206929:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098962.207727:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098962.207763:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098962.207926:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200704
00000100:00100000:0.0:1766098962.207967:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1325
00000100:00100000:0.0:1766098962.207973:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417200704:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098962.208071:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098962.208133:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417200704:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 160us (370us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098962.208142:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1325
00000100:00100000:3.0:1766098962.208422:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200576
00000100:00100000:2.0:1766098962.208789:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200640
00000100:00100000:2.0:1766098962.208823:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2165
00000100:00100000:2.0:1766098962.208827:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417200640:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098962.208871:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417200640:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 43us (1950us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098962.208877:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2165
00000100:00100000:1.0:1766098962.209620:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1472
00000100:00100000:1.0:1766098962.209626:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417200576:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098962.209702:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417200576:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (1978us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098962.209708:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1472
00010000:00010000:2.0:1766098962.301720:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098962.877686:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098962.877693:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098962.877696:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098962.877704:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3466
00000100:00080000:0.0:1766098962.877708:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098962.877714:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098962.877737:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000aa4b7e05 x1851885432898240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098962.877789:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006dd103c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432898304:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098962.877867:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098962.877901:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098962.877906:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc68e892 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432898176:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098962.877933:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098962.877971:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432898240:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098962.878006:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098962.878017:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898304
00000100:00100000:0.0:1766098962.878040:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898176
00000100:00100000:2.0:1766098962.878066:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2166
00000100:00100000:2.0:1766098962.878073:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432898304:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098962.878120:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898240
00000100:00100000:2.0:1766098962.878141:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432898304:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (269us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098962.878153:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1326
00000100:00100000:1.0:1766098962.878157:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432898240:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098962.878263:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098962.878278:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432898368:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098962.878298:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432898240:12345-0@lo:8:kworker/u8:0.0 Request processed in 140us (293us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098962.878303:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098962.878305:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1326
00000100:00100000:1.0:1766098962.878318:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898368
00000100:00100000:3.0:1766098962.878320:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766098962.878325:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432898112:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098962.878352:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098962.878377:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898112
00000100:00100000:3.0:1766098962.878403:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1473
00000100:00100000:3.0:1766098962.878406:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432898112:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098962.878437:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432898112:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (85us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098962.878455:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1473
00000100:00100000:1.0:1766098962.878511:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1000
00000100:00100000:1.0:1766098962.878515:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432898368:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098962.878522:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432898112:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098962.878556:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432898368:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (252us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098962.878562:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1000
00010000:00080000:3.0:1766098962.878565:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a3227a74 x1851885432898368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098969 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098962.878575:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432898368:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098962.878592:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098962.878596:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098962.878599:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432898240:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098962.878888:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2167
00000100:00100000:2.0:1766098962.878912:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2166
00000100:00100000:0.0:1766098962.878953:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432898176:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098962.878990:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432898176:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (1057us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098962.878996:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2167
00000100:00100000:0.0:1766098962.879025:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc68e892 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432898176:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098962.879034:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006dd103c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432898304:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098963.327533:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098964.352694:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098965.376974:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098966.397920:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098967.325740:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098967.326804:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098967.326861:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200832
00000100:00100000:1.0:1766098967.326914:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2168
00000100:00100000:1.0:1766098967.326920:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417200832:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098967.327044:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417200832:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 124us (1313us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098967.327052:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2168
00000100:00100000:0.0:1766098967.327360:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200768
00000100:00100000:0.0:1766098967.327392:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1474
00000100:00100000:0.0:1766098967.327397:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417200768:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766098967.327448:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417200768:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 50us (647us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098967.327455:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1474
00000100:00100000:3.0:1766098967.329922:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098967.330061:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200896
00000100:00100000:0.0:1766098967.330108:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1327
00000100:00100000:0.0:1766098967.330115:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417200896:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766098967.330232:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098967.330300:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417200896:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 183us (385us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098967.330309:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1327
00010000:00010000:0.0:1766098967.421773:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766098967.998233:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766098967.998239:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766098967.998243:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766098967.998251:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3471
00000100:00100000:1.0:1766098967.998252:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766098967.998255:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766098967.998261:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098967.998267:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432898432:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098967.998284:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005fa10d21 x1851885432898560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766098967.998335:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098967.998341:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432898496:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098967.998375:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432898688:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098967.998382:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098967.998414:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098967.998422:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432898624:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098967.998423:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098967.998455:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098967.998458:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432898560:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098967.998520:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898496
00000100:00100000:3.0:1766098967.998528:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898624
00000100:00100000:3.0:1766098967.998568:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2169
00000100:00100000:3.0:1766098967.998573:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432898624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098967.998605:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2170
00000100:00100000:2.0:1766098967.998610:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb97adba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432898496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098967.998680:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898432
00000100:00100000:0.0:1766098967.998703:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1475
00000100:00100000:0.0:1766098967.998707:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432898432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098967.998806:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dbc67d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432898624:12345-0@lo:400:kworker/u8:3.0 Request processed in 231us (352us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098967.998814:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2169
00000100:00100000:2.0:1766098967.998910:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb97adba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432898496:12345-0@lo:400:kworker/u8:3.0 Request processed in 298us (528us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098967.998918:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2170
00000100:00100000:2.0:1766098967.998965:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ea4f334 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432898624:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098967.998985:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432898496:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098967.999022:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898688
00000100:00100000:2.0:1766098967.999042:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1001
00000100:00100000:2.0:1766098967.999045:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432898688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098967.999074:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432898688:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (653us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098967.999079:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1001
00000100:00100000:1.0:1766098967.999465:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098967.999496:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432898432:12345-0@lo:400:kworker/u8:3.0 Request processed in 768us (1143us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098967.999504:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1475
00010000:00080000:0.0:1766098967.999616:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cc7d0c6a x1851885432898688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098974 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766098967.999626:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc7d0c6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432898688:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098967.999633:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5762093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432898432:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098968.001436:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898560
00000100:00100000:2.0:1766098968.001493:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1328
00000100:00100000:2.0:1766098968.001498:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432898560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766098968.001596:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098968.001672:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432898560:12345-0@lo:8:kworker/u8:3.0 Request processed in 173us (2209us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098968.001680:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1328
00000100:00080000:1.0:1766098968.001844:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098968.001849:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098968.001853:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432898560:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766098968.445746:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098969.472437:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098970.496636:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098971.519475:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098972.445665:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098972.445718:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098972.445762:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098972.445876:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417200960
00000100:00100000:0.0:1766098972.445945:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1476
00000100:00100000:0.0:1766098972.445953:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417200960:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766098972.446034:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417200960:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 82us (382us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098972.446042:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1476
00000100:00100000:2.0:1766098972.446269:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201024
00000100:00100000:2.0:1766098972.446309:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2171
00000100:00100000:2.0:1766098972.446313:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417201024:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098972.446349:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417201024:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 36us (588us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098972.446356:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2171
00000100:00100000:3.0:1766098972.446555:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201088
00000100:00100000:3.0:1766098972.446587:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1329
00000100:00100000:3.0:1766098972.446591:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c0ecbea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417201088:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766098972.446807:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098972.446878:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c0ecbea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417201088:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 286us (1160us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098972.446885:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1329
00010000:00010000:0.0:1766098972.544001:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098973.118104:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432898752:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098973.118183:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098973.118246:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898752
00000100:00100000:1.0:1766098973.118300:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1477
00000100:00100000:1.0:1766098973.118308:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432898752:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098973.118359:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432898752:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (177us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098973.118376:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1477
00000100:00100000:1.0:1766098973.118435:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432898752:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766098973.121656:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098973.121664:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098973.121667:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098973.121674:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3476
00000100:00080000:3.0:1766098973.121677:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098973.121682:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098973.121700:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001a3c97aa x1851885432898880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098973.121810:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432898944:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098973.121863:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098973.121889:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766098973.121893:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432898816:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098973.121909:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098973.121938:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432898880:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766098973.121962:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098973.121987:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766098973.121994:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432899008:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098973.122015:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098973.122040:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898880
00000100:00100000:3.0:1766098973.122064:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899008
00000100:00100000:3.0:1766098973.122083:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1002
00000100:00100000:3.0:1766098973.122088:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432899008:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766098973.122125:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432899008:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (110us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098973.122131:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1002
00000100:00100000:3.0:1766098973.122143:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1330
00000100:00100000:3.0:1766098973.122146:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432898880:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766098973.122234:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098973.122256:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432898880:12345-0@lo:8:kworker/u8:3.0 Request processed in 109us (294us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098973.122262:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1330
00000100:00080000:3.0:1766098973.122289:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098973.122292:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098973.122295:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432898880:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766098973.122313:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fcd4deb0 x1851885432899008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098980 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766098973.122321:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcd4deb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432899008:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098973.124488:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898944
00000100:00100000:0.0:1766098973.124567:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432898816
00000100:00100000:0.0:1766098973.124595:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2172
00000100:00100000:0.0:1766098973.124601:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432898944:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098973.124713:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432898944:12345-0@lo:400:kworker/u8:3.0 Request processed in 112us (2850us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098973.124722:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2172
00000100:00100000:0.0:1766098973.124741:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2173
00000100:00100000:0.0:1766098973.124745:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432898816:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098973.125232:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003cd96141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432898944:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098973.125324:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432898816:12345-0@lo:400:kworker/u8:3.0 Request processed in 578us (3414us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098973.125332:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2173
00000100:00100000:2.0:1766098973.125888:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432898816:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098973.565805:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098974.591345:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098975.613899:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098976.637668:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098977.567048:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098977.567086:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098977.567162:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098977.567254:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201280
00000100:00100000:2.0:1766098977.567297:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1331
00000100:00100000:2.0:1766098977.567303:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417201280:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766098977.567400:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766098977.567458:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417201280:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 155us (416us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766098977.567465:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1331
00000100:00100000:1.0:1766098977.567616:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201216
00000100:00100000:1.0:1766098977.567655:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2174
00000100:00100000:1.0:1766098977.567658:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417201216:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098977.567697:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417201216:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 38us (611us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098977.567717:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2174
00000100:00100000:3.0:1766098977.568377:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201152
00000100:00100000:3.0:1766098977.568500:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1478
00000100:00100000:3.0:1766098977.568507:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417201152:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766098977.568587:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417201152:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 80us (1424us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098977.568594:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1478
00010000:00010000:2.0:1766098977.661714:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766098978.239110:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432899072:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098978.239175:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098978.239468:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899072
00000100:00100000:2.0:1766098978.239529:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1479
00000100:00100000:2.0:1766098978.239535:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432899072:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766098978.239643:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098978.239648:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098978.239651:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098978.239656:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3481
00000100:00080000:1.0:1766098978.239658:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098978.239662:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098978.239680:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002810083f x1851885432899200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766098978.239835:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002810083f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432899200:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098978.239868:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098978.240582:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899200
00000100:00100000:3.0:1766098978.240608:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1332
00000100:00100000:3.0:1766098978.240612:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432899200:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766098978.240708:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098978.240743:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432899200:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (874us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098978.240749:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1332
00000100:00080000:3.0:1766098978.240778:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098978.240781:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098978.240784:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002810083f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432899200:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098978.240866:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432899328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.240889:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098978.240907:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432899136:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.240926:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098978.240939:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098978.240942:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432899264:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.240957:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098978.240987:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899136
00000100:00100000:3.0:1766098978.241007:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899264
00000100:00100000:3.0:1766098978.241019:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2175
00000100:00100000:3.0:1766098978.241022:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432899136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.241050:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003cd96141 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432899136:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (125us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098978.241055:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2175
00000100:00100000:3.0:1766098978.241065:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2176
00000100:00100000:3.0:1766098978.241067:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432899264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.241083:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fcd4deb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432899264:12345-0@lo:400:kworker/u8:0.0 Request processed in 15us (126us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098978.241087:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2176
00000100:00100000:3.0:1766098978.241114:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e445b90d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432899264:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.241123:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003a108df5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432899136:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.241143:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899328
00000100:00100000:3.0:1766098978.241159:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1003
00000100:00100000:3.0:1766098978.241162:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432899328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098978.241186:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432899328:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (298us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098978.241192:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1003
00010000:00080000:3.0:1766098978.241210:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c8b37342 x1851885432899328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098985 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766098978.241233:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432899328:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098978.241352:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432899072:12345-0@lo:400:kworker/u8:0.0 Request processed in 1816us (2177us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098978.241360:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1479
00000100:00100000:0.0:1766098978.241432:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432899072:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766098978.688380:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098979.709735:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098980.733778:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098981.758829:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766098982.684615:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098982.684800:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098982.684809:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201408
00000100:00100000:1.0:1766098982.684882:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2177
00000100:00100000:1.0:1766098982.684890:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5762093 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417201408:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766098982.684968:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5762093 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417201408:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (370us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098982.684976:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2177
00000100:00100000:2.0:1766098982.685896:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098982.685952:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201472
00000100:00100000:0.0:1766098982.685994:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1333
00000100:00100000:0.0:1766098982.686000:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417201472:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:3.0:1766098982.686025:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201344
00000100:00100000:3.0:1766098982.686069:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1480
00000100:00100000:3.0:1766098982.686081:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417201344:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00010000:02020000:0.0:1766098982.686106:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766098982.686133:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417201344:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 51us (1333us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098982.686141:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417201472:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 139us (247us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766098982.686144:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1480
00000100:00100000:0.0:1766098982.686153:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1333
00010000:00010000:0.0:1766098982.781747:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766098983.359066:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766098983.359073:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766098983.359076:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766098983.359084:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3486
00000100:00080000:1.0:1766098983.359088:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766098983.359093:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766098983.359111:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000142c5414 x1851885432899520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098983.359196:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039d784a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432899648:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098983.359255:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098983.359317:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098983.359325:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432899392:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098983.359352:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098983.359386:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899392
00000100:00100000:2.0:1766098983.359422:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1481
00000100:00100000:2.0:1766098983.359427:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432899392:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766098983.359469:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432899392:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (116us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098983.359501:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1481
00000100:00100000:2.0:1766098983.359543:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432899392:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359613:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432899456:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359645:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098983.359677:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766098983.359682:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432899584:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359702:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098983.359729:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000142c5414 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432899520:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359750:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766098983.359780:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899456
00000100:00100000:3.0:1766098983.359802:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899584
00000100:00100000:3.0:1766098983.359819:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2178
00000100:00100000:3.0:1766098983.359822:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432899456:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359852:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432899456:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (207us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098983.359857:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2178
00000100:00100000:3.0:1766098983.359870:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2179
00000100:00100000:3.0:1766098983.359873:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432899584:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359895:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432899584:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (192us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098983.359901:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2179
00000100:00100000:3.0:1766098983.359929:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432899584:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766098983.359956:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432899456:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098983.360060:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899648
00000100:00100000:0.0:1766098983.360081:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899520
00000100:00100000:0.0:1766098983.360092:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004
00000100:00100000:0.0:1766098983.360095:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432899648:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098983.360129:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432899648:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (875us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098983.360134:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004
00000100:00100000:0.0:1766098983.360146:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1334
00000100:00100000:0.0:1766098983.360149:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432899520:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766098983.360243:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098983.360266:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432899520:12345-0@lo:8:kworker/u8:0.0 Request processed in 116us (516us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098983.360272:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1334
00010000:00080000:0.0:1766098983.360303:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000039d784a6 x1851885432899648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098990 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766098983.360312:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039d784a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432899648:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766098983.360654:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766098983.360658:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766098983.360662:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000142c5414 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432899520:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766098983.805689:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098984.829791:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098985.854306:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098986.877855:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098987.808556:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098987.809071:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098987.809115:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098987.809154:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201536
00000100:00100000:3.0:1766098987.809206:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1482
00000100:00100000:3.0:1766098987.809212:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417201536:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766098987.809283:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417201536:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 70us (214us total) trans 0 rc 0/0
00000100:00100000:3.0:1766098987.809290:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1482
00000100:00100000:0.0:1766098987.809396:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201664
00000100:00100000:0.0:1766098987.809421:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1335
00000100:00100000:0.0:1766098987.809434:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417201664:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098987.809557:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098987.809606:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417201664:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 171us (1058us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098987.809613:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1335
00000100:00100000:2.0:1766098987.809740:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201600
00000100:00100000:2.0:1766098987.809782:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2180
00000100:00100000:2.0:1766098987.809787:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d784a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417201600:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098987.809833:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d784a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417201600:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (718us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098987.809847:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2180
00010000:00010000:0.0:1766098987.901859:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766098988.479087:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432899712:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098988.479152:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:3.0:1766098988.479220:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766098988.479225:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766098988.479228:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766098988.479232:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3491
00000100:00080000:3.0:1766098988.479235:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766098988.479240:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766098988.479269:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000fe333148 x1851885432899840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766098988.479601:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432899776:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098988.479634:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098988.479657:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766098988.479662:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432899904:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098988.479681:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098988.479719:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899712
00000100:00100000:2.0:1766098988.479757:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1483
00000100:00100000:2.0:1766098988.479762:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432899712:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480118:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432899968:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480153:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098988.480184:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899776
00000100:00100000:0.0:1766098988.480211:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899904
00000100:00100000:0.0:1766098988.480230:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2181
00000100:00100000:0.0:1766098988.480234:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432899776:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480297:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc7d0c6a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432899776:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (663us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098988.480305:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2181
00000100:00100000:0.0:1766098988.480323:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2182
00000100:00100000:0.0:1766098988.480327:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432899904:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480353:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ea4f334 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432899904:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (672us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098988.480359:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2182
00000100:00100000:0.0:1766098988.480393:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432899904:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480404:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432899776:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480436:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899968
00000100:00100000:0.0:1766098988.480458:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1005
00000100:00100000:0.0:1766098988.480493:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432899968:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480530:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432899968:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (377us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098988.480537:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1005
00010000:00080000:0.0:1766098988.480560:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000a6b62b0 x1851885432899968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766098995 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766098988.480571:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432899968:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480608:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432899840:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098988.480631:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098988.480657:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432899840
00000100:00100000:0.0:1766098988.480673:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1336
00000100:00100000:0.0:1766098988.480676:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432899840:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098988.480766:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098988.480796:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432899840:12345-0@lo:8:kworker/u8:3.0 Request processed in 118us (165us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098988.480802:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1336
00000100:00080000:0.0:1766098988.480825:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098988.480828:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098988.480831:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432899840:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766098988.481302:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432899712:12345-0@lo:400:kworker/u8:3.0 Request processed in 1537us (2149us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098988.481311:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1483
00000100:00100000:1.0:1766098988.481829:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432899712:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766098988.925722:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098989.949764:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098990.974633:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098991.997662:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098992.925398:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098992.925442:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766098992.925628:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098992.925694:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201728
00000100:00100000:2.0:1766098992.925737:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1484
00000100:00100000:2.0:1766098992.925744:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417201728:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098992.925809:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417201728:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 64us (366us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098992.925830:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1484
00000100:00100000:0.0:1766098992.926455:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201856
00000100:00100000:1.0:1766098992.926539:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201792
00000100:00100000:1.0:1766098992.926576:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2183
00000100:00100000:1.0:1766098992.926580:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047fe747c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417201792:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098992.926627:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047fe747c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417201792:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 35us (989us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098992.926633:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2183
00000100:00100000:0.0:1766098992.926863:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1337
00000100:00100000:0.0:1766098992.926869:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417201856:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098992.926961:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098992.927005:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417201856:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 136us (1616us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098992.927012:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1337
00010000:00010000:2.0:1766098993.023374:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098993.599458:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098993.599466:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098993.599469:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098993.599498:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3496
00000100:00080000:0.0:1766098993.599502:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098993.599508:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098993.599530:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000045df1f7c x1851885432900160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098993.599582:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432900096:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.599642:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098993.599675:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766098993.599680:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432900224:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.599721:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098993.599751:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432900160:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098993.599778:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098993.599812:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900096
00000100:00100000:0.0:1766098993.599847:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900224
00000100:00100000:0.0:1766098993.599866:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2184
00000100:00100000:0.0:1766098993.599871:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432900096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.599916:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432900096:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (274us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098993.599923:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2184
00000100:00100000:0.0:1766098993.599949:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2185
00000100:00100000:0.0:1766098993.599954:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005aa8e356 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432900224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.599980:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005aa8e356 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432900224:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (260us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098993.599985:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2185
00000100:00100000:0.0:1766098993.600003:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900160
00000100:00100000:0.0:1766098993.600034:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1338
00000100:00100000:0.0:1766098993.600038:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042efdcc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432900160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766098993.600126:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098993.600154:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042efdcc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432900160:12345-0@lo:8:kworker/u8:3.0 Request processed in 115us (376us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098993.600160:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1338
00000100:00080000:0.0:1766098993.600201:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766098993.600204:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766098993.600208:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432900160:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766098993.600247:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432900224:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.600255:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432900096:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098993.600355:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432900288:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098993.600385:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766098993.600422:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766098993.600428:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432900032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766098993.600451:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098993.600625:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900032
00000100:00100000:1.0:1766098993.600662:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1485
00000100:00100000:1.0:1766098993.600666:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432900032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766098993.600705:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432900032:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (253us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098993.600712:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1485
00000100:00100000:1.0:1766098993.600744:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432900032:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.600843:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900288
00000100:00100000:0.0:1766098993.601795:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1006
00000100:00100000:0.0:1766098993.601802:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432900288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766098993.601860:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432900288:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (1474us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098993.601866:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1006
00010000:00080000:2.0:1766098993.602245:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004039bd81 x1851885432900288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099000 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766098993.602258:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432900288:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766098994.045810:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098995.072638:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766098996.094964:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766098997.118913:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766098998.045785:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766098998.047231:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201920
00000100:00100000:2.0:1766098998.047286:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1486
00000100:00100000:2.0:1766098998.047292:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417201920:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766098998.047363:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417201920:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 71us (1585us total) trans 0 rc 0/0
00000100:00100000:2.0:1766098998.047370:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1486
00000100:00100000:3.0:1766098998.053825:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766098998.054560:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202048
00000100:00100000:0.0:1766098998.054608:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1339
00000100:00100000:0.0:1766098998.054614:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417202048:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766098998.054704:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766098998.054761:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417202048:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 147us (943us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766098998.054767:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1339
00000100:00100000:3.0:1766098998.060648:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766098998.060745:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417201984
00000100:00100000:1.0:1766098998.060797:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2186
00000100:00100000:1.0:1766098998.060802:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000923c0bd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417201984:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766098998.060869:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000923c0bd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417201984:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 66us (229us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098998.060876:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2186
00010000:00010000:2.0:1766098998.142457:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766098998.719088:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766098998.719095:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766098998.719098:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766098998.719105:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3502
00000100:00080000:0.0:1766098998.719108:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766098998.719113:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766098998.719133:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f0765a5d x1851885432900480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766098998.719186:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002bc5bf74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432900544:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098998.719246:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098998.719275:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766098998.719281:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000070350b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432900416:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098998.719303:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098998.719325:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0765a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432900480:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766098998.719348:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766098998.719380:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900544
00000100:00100000:0.0:1766098998.719416:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900416
00000100:00100000:0.0:1766098998.719452:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2187
00000100:00100000:0.0:1766098998.719458:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011a432a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432900544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098998.719540:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011a432a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432900544:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (294us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098998.719548:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2187
00000100:00100000:0.0:1766098998.719566:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2188
00000100:00100000:0.0:1766098998.719569:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006f41af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432900416:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098998.719592:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006f41af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432900416:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (289us total) trans 0 rc 0/0
00000100:00100000:0.0:1766098998.719598:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2188
00000100:00100000:0.0:1766098998.719645:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000070350b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432900416:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766098998.719656:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002bc5bf74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432900544:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098998.719745:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900480
00000100:00100000:1.0:1766098998.719781:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1340
00000100:00100000:1.0:1766098998.719786:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5512bcd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432900480:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766098998.719877:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766098998.719912:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5512bcd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432900480:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (563us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766098998.719918:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1340
00000100:00080000:1.0:1766098998.719941:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766098998.719945:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766098998.719948:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0765a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432900480:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766098998.719964:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097631d91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432900352:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098998.719992:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098998.720008:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766098998.720013:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432900608:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098998.720032:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766098998.720055:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900352
00000100:00100000:1.0:1766098998.720076:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1487
00000100:00100000:1.0:1766098998.720080:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432900352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098998.720109:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432900352:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (117us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098998.720114:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1487
00000100:00100000:1.0:1766098998.720144:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097631d91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432900352:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098998.720160:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900608
00000100:00100000:1.0:1766098998.720193:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1007
00000100:00100000:1.0:1766098998.720196:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432900608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766098998.720236:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432900608:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (204us total) trans 0 rc 0/0
00000100:00100000:1.0:1766098998.720241:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1007
00010000:00080000:1.0:1766098998.720267:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002df41de4 x1851885432900608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099005 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766098998.720276:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432900608:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766098999.166001:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099000.189734:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099001.216754:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099002.237734:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099003.166101:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099003.168382:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202176
00000100:00100000:1.0:1766099003.168455:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2189
00000100:00100000:1.0:1766099003.168463:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027be1d19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417202176:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099003.168577:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027be1d19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417202176:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 114us (2483us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099003.168584:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2189
00000100:00100000:3.0:1766099003.173210:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099003.173259:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099003.174142:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202240
00000100:00100000:0.0:1766099003.174201:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1341
00000100:00100000:0.0:1766099003.174208:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417202240:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099003.174313:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099003.174385:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417202240:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 176us (1182us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099003.174392:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1341
00000100:00100000:2.0:1766099003.174498:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202112
00000100:00100000:2.0:1766099003.174540:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1488
00000100:00100000:2.0:1766099003.174544:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417202112:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099003.174576:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417202112:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 31us (1317us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099003.174582:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1488
00010000:00010000:0.0:1766099003.262507:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099003.844320:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099003.844327:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099003.844330:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099003.844339:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3507
00000100:00080000:1.0:1766099003.844342:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099003.844347:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099003.844369:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000844e5d17 x1851885432900800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099003.844432:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b79b30ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432900928:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.844512:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099003.844547:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099003.844552:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432900672:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.844589:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099003.844613:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051f529aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432900864:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.844636:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099003.844651:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099003.844655:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9122345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432900736:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.844674:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099003.844694:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000844e5d17 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432900800:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099003.844714:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099003.844746:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900928
00000100:00100000:1.0:1766099003.844779:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900800
00000100:00100000:1.0:1766099003.844798:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1008
00000100:00100000:1.0:1766099003.844803:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af3b0884 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432900928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.844852:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af3b0884 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432900928:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (340us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099003.844860:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1008
00000100:00100000:1.0:1766099003.844876:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1342
00000100:00100000:1.0:1766099003.844880:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5c3581c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432900800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099003.844972:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099003.845002:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5c3581c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432900800:12345-0@lo:8:kworker/u8:0.0 Request processed in 121us (288us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099003.845009:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1342
00000100:00100000:1.0:1766099003.845026:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900672
00000100:00100000:1.0:1766099003.845049:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1489
00000100:00100000:1.0:1766099003.845052:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432900672:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.845080:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432900672:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (491us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099003.845094:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1489
00000100:00100000:1.0:1766099003.845121:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900864
00000100:00100000:1.0:1766099003.845155:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900736
00000100:00100000:1.0:1766099003.845174:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2190
00000100:00100000:1.0:1766099003.845178:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005b7ae9f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432900864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.845211:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005b7ae9f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432900864:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (575us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099003.845217:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2190
00000100:00100000:1.0:1766099003.845229:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2191
00000100:00100000:1.0:1766099003.845233:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009041fdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432900736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.845258:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009041fdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432900736:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (584us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099003.845263:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2191
00000100:00100000:1.0:1766099003.845299:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432900672:0@lo:400:kworker/u8:0.0
00010000:00080000:1.0:1766099003.845311:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b79b30ea x1851885432900928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099010 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099003.845322:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b79b30ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432900928:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766099003.845351:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099003.845355:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099003.845358:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000844e5d17 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432900800:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099003.845386:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9122345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432900736:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099003.845395:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051f529aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432900864:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766099004.285703:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099005.309683:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099006.333967:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099007.358977:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099008.287186:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099008.288721:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099008.289882:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202432
00000100:00100000:1.0:1766099008.289969:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1343
00000100:00100000:1.0:1766099008.289976:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417202432:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099008.290088:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099008.290190:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417202432:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 214us (3015us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099008.290199:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1343
00000100:00100000:3.0:1766099008.290613:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099008.291562:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202368
00000100:00100000:0.0:1766099008.291643:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2192
00000100:00100000:0.0:1766099008.291650:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417202368:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099008.291737:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417202368:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 86us (3022us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099008.291744:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2192
00000100:00100000:2.0:1766099008.292204:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202304
00000100:00100000:2.0:1766099008.292239:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1490
00000100:00100000:2.0:1766099008.292244:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202304:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099008.292273:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202304:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 28us (1662us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099008.292279:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1490
00010000:00010000:2.0:1766099008.382658:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099008.959423:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1766099008.959423:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051f529aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432900992:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099008.959433:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099008.959438:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099008.959444:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3512
00000100:00080000:1.0:1766099008.959447:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099008.959454:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099008.959501:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000844e5d17 x1851885432901120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099008.959548:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099008.959570:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f0d53e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432901248:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099008.959623:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099008.959638:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432900992
00000100:00100000:1.0:1766099008.959684:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9122345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432901056:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099008.959708:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1491
00000100:00100000:3.0:1766099008.959717:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432900992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099008.959728:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099008.959755:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099008.959763:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b79b30ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432901184:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099008.959797:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432900992:12345-0@lo:400:kworker/u8:3.0 Request processed in 77us (251us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099008.959798:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099008.959808:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1491
00000100:00100000:1.0:1766099008.959835:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000844e5d17 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432901120:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099008.959862:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099008.959887:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901248
00000100:00100000:0.0:1766099008.959924:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901120
00000100:00100000:0.0:1766099008.959940:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1009
00000100:00100000:0.0:1766099008.959944:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3e3a280 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432901248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099008.959982:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3e3a280 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432901248:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (364us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099008.959988:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1009
00000100:00100000:0.0:1766099008.960005:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1344
00000100:00100000:0.0:1766099008.960008:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c92908a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432901120:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099008.960133:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099008.960171:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c92908a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432901120:12345-0@lo:8:kworker/u8:3.0 Request processed in 161us (309us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099008.960178:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1344
00010000:00080000:0.0:1766099008.960408:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006f0d53e0 x1851885432901248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099015 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099008.960422:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f0d53e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432901248:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099008.960452:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099008.960456:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099008.960458:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000844e5d17 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432901120:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099008.960833:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901184
00000100:00100000:2.0:1766099008.960839:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901056
00000100:00100000:1.0:1766099008.960894:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2193
00000100:00100000:1.0:1766099008.960900:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5214f23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432901184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099008.960906:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051f529aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432900992:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099008.960950:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5214f23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432901184:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (1152us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099008.960958:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2193
00000100:00100000:3.0:1766099008.960969:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b79b30ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432901184:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099008.962462:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2194
00000100:00100000:2.0:1766099008.962473:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e539110 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432901056:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099008.962594:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e539110 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432901056:12345-0@lo:400:kworker/u8:3.0 Request processed in 121us (2870us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099008.962601:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2194
00000100:00100000:3.0:1766099008.962782:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9122345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432901056:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099009.405824:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099010.429880:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099011.458230:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099012.479188:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099013.405379:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099013.405438:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099013.405474:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202496
00000100:00100000:3.0:1766099013.405546:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099013.405787:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202624
00000100:00100000:1.0:1766099013.405854:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1345
00000100:00100000:1.0:1766099013.405863:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417202624:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099013.405988:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099013.406078:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417202624:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 205us (630us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099013.406086:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1345
00000100:00100000:0.0:1766099013.406172:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1492
00000100:00100000:0.0:1766099013.406176:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202496:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099013.406213:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202496:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 37us (846us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099013.406220:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1492
00000100:00100000:2.0:1766099013.406547:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202560
00000100:00100000:2.0:1766099013.406591:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2195
00000100:00100000:2.0:1766099013.406596:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000edd8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417202560:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099013.406637:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000edd8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417202560:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 40us (1093us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099013.406643:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2195
00010000:00010000:2.0:1766099013.504499:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099014.077890:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099014.077900:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099014.077904:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099014.077914:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3517
00000100:00080000:1.0:1766099014.077918:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099014.077925:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099014.077951:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000066675ebb x1851885432901440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099014.078003:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f90f2e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432901568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078116:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099014.078156:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099014.078162:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c15c4086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432901312:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078190:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099014.078236:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fdac9781 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432901504:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078266:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099014.078282:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099014.078286:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a809e10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432901376:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078306:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099014.078327:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000066675ebb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432901440:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078349:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099014.078388:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901568
00000100:00100000:1.0:1766099014.078434:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901440
00000100:00100000:1.0:1766099014.078455:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1010
00000100:00100000:1.0:1766099014.078461:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432901568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078550:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432901568:12345-0@lo:400:kworker/u8:3.0 Request processed in 87us (435us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099014.078558:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1010
00000100:00100000:1.0:1766099014.078579:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1346
00000100:00100000:1.0:1766099014.078583:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba35ba7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432901440:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099014.078591:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901504
00010000:02020000:1.0:1766099014.078676:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099014.078702:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba35ba7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432901440:12345-0@lo:8:kworker/u8:3.0 Request processed in 119us (354us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099014.078707:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1346
00000100:00100000:3.0:1766099014.078737:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901376
00000100:00080000:1.0:1766099014.078747:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099014.078751:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099014.078756:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000066675ebb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432901440:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099014.078771:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2196
00000100:00100000:3.0:1766099014.078778:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000763a5308 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432901504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078786:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2197
00000100:00100000:1.0:1766099014.078792:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073d97e02 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432901376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078833:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073d97e02 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432901376:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (527us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099014.078840:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2197
00000100:00100000:1.0:1766099014.078870:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a809e10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432901376:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766099014.078893:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f90f2e7d x1851885432901568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099021 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099014.078901:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f90f2e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432901568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078943:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901312
00000100:00100000:1.0:1766099014.078967:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1493
00000100:00100000:1.0:1766099014.078970:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432901312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099014.078996:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432901312:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (806us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099014.079001:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1493
00000100:00100000:1.0:1766099014.079029:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c15c4086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432901312:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099014.081742:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000763a5308 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432901504:12345-0@lo:400:kworker/u8:3.0 Request processed in 2953us (3464us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099014.081784:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2196
00000100:00100000:1.0:1766099014.081858:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fdac9781 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432901504:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099014.525760:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099015.553121:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099016.578889:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099017.610164:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099018.524998:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099018.525348:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202816
00000100:00100000:2.0:1766099018.525448:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1347
00000100:00100000:2.0:1766099018.525457:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417202816:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099018.525617:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099018.525709:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417202816:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 252us (722us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099018.525719:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1347
00000100:00100000:1.0:1766099018.529721:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099018.531162:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099018.531286:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202752
00000100:00100000:0.0:1766099018.531392:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202688
00000100:00100000:0.0:1766099018.531432:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1494
00000100:00100000:0.0:1766099018.531441:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202688:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099018.531551:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202688:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 110us (1841us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099018.531560:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1494
00000100:00100000:3.0:1766099018.531688:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2198
00000100:00100000:3.0:1766099018.531694:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fdac9781 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417202752:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099018.531732:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fdac9781 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417202752:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 38us (577us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099018.531738:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2198
00010000:00010000:0.0:1766099018.621876:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099019.197910:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099019.197920:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099019.197925:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099019.197936:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3522
00000100:00080000:2.0:1766099019.197940:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099019.197947:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099019.197974:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000affb1b2b x1851885432901760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099019.198103:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432901760:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198190:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099019.198243:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432901824:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198270:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099019.198288:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099019.198292:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432901696:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198330:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099019.198364:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901824
00000100:00100000:1.0:1766099019.198405:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901696
00000100:00100000:1.0:1766099019.198422:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2199
00000100:00100000:1.0:1766099019.198427:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f90f2e7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432901824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198496:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f90f2e7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432901824:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (206us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099019.198503:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2199
00000100:00100000:1.0:1766099019.198521:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2200
00000100:00100000:1.0:1766099019.198524:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a809e10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432901696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198543:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a809e10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432901696:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (213us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099019.198548:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2200
00000100:00100000:1.0:1766099019.198582:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000025c2fc30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432901696:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198590:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a6b62b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432901824:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198621:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432901632:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198646:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099019.198662:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099019.198665:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432901888:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198681:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099019.198702:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901632
00000100:00100000:1.0:1766099019.198723:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1495
00000100:00100000:1.0:1766099019.198726:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432901632:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198751:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432901632:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (105us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099019.198764:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1495
00000100:00100000:1.0:1766099019.198793:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901760
00000100:00100000:1.0:1766099019.198807:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901888
00000100:00100000:1.0:1766099019.198822:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1011
00000100:00100000:1.0:1766099019.198825:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066675ebb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432901888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.198851:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066675ebb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432901888:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (169us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099019.198857:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1011
00000100:00100000:1.0:1766099019.198868:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1348
00000100:00100000:1.0:1766099019.198871:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432901760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099019.198962:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099019.198989:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432901760:12345-0@lo:8:kworker/u8:2.0 Request processed in 117us (801us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099019.198995:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1348
00000100:00080000:1.0:1766099019.199019:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099019.199022:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099019.199036:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000affb1b2b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432901760:0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766099019.199061:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005fa10d21 x1851885432901888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099026 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099019.199069:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432901888:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099019.199076:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432901632:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099019.645797:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099020.669833:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099021.693831:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099022.718065:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099023.648246:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099023.648302:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099023.648440:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203008
00000100:00100000:2.0:1766099023.648527:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1349
00000100:00100000:2.0:1766099023.648536:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000305829f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417203008:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099023.648636:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099023.648719:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000305829f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417203008:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 183us (487us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099023.648727:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1349
00000100:00100000:1.0:1766099023.650107:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202944
00000100:00100000:1.0:1766099023.650175:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2201
00000100:00100000:1.0:1766099023.650181:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417202944:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099023.650304:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417202944:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 122us (2002us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099023.650312:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2201
00000100:00100000:3.0:1766099023.650755:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099023.650821:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417202880
00000100:00100000:0.0:1766099023.650863:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1496
00000100:00100000:0.0:1766099023.650868:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202880:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099023.650911:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417202880:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 42us (158us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099023.650917:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1496
00010000:00010000:2.0:1766099023.741822:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099024.318193:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432901952:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099024.318297:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099024.318367:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432901952
00000100:00100000:0.0:1766099024.318419:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1497
00000100:00100000:0.0:1766099024.318425:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432901952:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099024.319822:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099024.319830:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099024.319834:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099024.319841:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3527
00000100:00080000:3.0:1766099024.319845:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099024.319851:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099024.319878:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007b5b6926 x1851885432902080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099024.319938:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432902208:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099024.320894:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432901952:12345-0@lo:400:kworker/u8:3.0 Request processed in 2462us (2593us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099024.320928:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1497
00000100:00100000:0.0:1766099024.321026:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000068a35948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432902016:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099024.321086:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099024.321119:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099024.321126:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432902144:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099024.321147:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099024.321228:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902016
00000100:00100000:0.0:1766099024.321269:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902144
00000100:00100000:0.0:1766099024.321288:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2202
00000100:00100000:0.0:1766099024.321292:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042efdcc3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432902016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099024.321343:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042efdcc3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432902016:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (259us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099024.321350:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2202
00000100:00100000:0.0:1766099024.321360:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2203
00000100:00100000:0.0:1766099024.321363:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5512bcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432902144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099024.321384:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5512bcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432902144:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (237us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099024.321389:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2203
00000100:00100000:0.0:1766099024.321449:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432902144:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099024.321460:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000068a35948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432902016:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099024.321552:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432902080:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099024.321590:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099024.321625:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902080
00000100:00100000:2.0:1766099024.321646:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1350
00000100:00100000:2.0:1766099024.321650:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432902080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099024.321752:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099024.321798:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432902080:12345-0@lo:8:kworker/u8:3.0 Request processed in 146us (206us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099024.321804:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1350
00000100:00100000:3.0:1766099024.322125:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099024.322235:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432901952:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099024.323081:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902208
00000100:00080000:0.0:1766099024.323341:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099024.323349:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099024.323356:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432902080:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099024.323700:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1012
00000100:00100000:2.0:1766099024.323708:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432902208:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099024.323786:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432902208:12345-0@lo:400:kworker/u8:3.0 Request processed in 79us (1662us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099024.323793:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1012
00010000:00080000:3.0:1766099024.323879:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bf96017c x1851885432902208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099031 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099024.323892:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432902208:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099024.765696:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099025.789770:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099026.814326:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099027.839185:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099028.765146:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099028.765205:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099028.765309:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099028.765362:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203200
00000100:00100000:2.0:1766099028.765409:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1351
00000100:00100000:2.0:1766099028.765417:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417203200:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099028.765568:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099028.765650:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417203200:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 232us (340us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099028.765676:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1351
00000100:00100000:1.0:1766099028.766057:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203136
00000100:00100000:1.0:1766099028.766108:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2204
00000100:00100000:1.0:1766099028.766113:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417203136:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099028.767163:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203072
00000100:00100000:3.0:1766099028.767227:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1498
00000100:00100000:3.0:1766099028.767234:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417203072:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099028.773658:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417203136:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 7532us (8513us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099028.773662:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417203072:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 6414us (8444us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099028.773686:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2204
00000100:00100000:3.0:1766099028.773694:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1498
00010000:00010000:2.0:1766099028.862088:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099029.437845:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000685e9d1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432902272:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099029.437850:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099029.437856:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099029.437860:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099029.437865:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3532
00000100:00080000:1.0:1766099029.437868:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099029.437873:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099029.437893:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000242dd738 x1851885432902400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099029.437916:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099029.437942:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cde33f60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432902336:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099029.437943:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099029.437960:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b4351f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432902528:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099029.438000:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099029.438001:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099029.438028:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099029.438062:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009fd582b8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432902464:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099029.438066:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902336
00000100:00100000:1.0:1766099029.438094:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099029.438133:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902528
00000100:00100000:1.0:1766099029.438178:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1013
00000100:00100000:1.0:1766099029.438185:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432902528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099029.438232:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432902528:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (236us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099029.438239:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1013
00000100:00100000:3.0:1766099029.438407:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902272
00000100:00100000:3.0:1766099029.438432:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1499
00000100:00100000:3.0:1766099029.438435:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432902272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099029.438464:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432902272:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (553us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099029.438470:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1499
00000100:00100000:2.0:1766099029.439087:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902464
00000100:00100000:1.0:1766099029.439096:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2205
00000100:00100000:1.0:1766099029.439103:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9122345 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432902336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099029.439118:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2206
00000100:00100000:2.0:1766099029.439123:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b79b30ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432902464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099029.439149:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9122345 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432902336:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (1154us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099029.439157:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2205
00000100:00100000:2.0:1766099029.439165:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b79b30ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432902464:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (1071us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099029.439172:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2206
00000100:00100000:1.0:1766099029.439197:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009fd582b8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432902464:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766099029.439215:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006b4351f3 x1851885432902528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099036 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099029.439221:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cde33f60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432902336:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099029.439232:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b4351f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432902528:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099029.439248:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000685e9d1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432902272:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099029.439348:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000242dd738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432902400:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099029.439379:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099029.439409:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902400
00000100:00100000:1.0:1766099029.439440:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1352
00000100:00100000:1.0:1766099029.439452:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432902400:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099029.439552:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099029.439599:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432902400:12345-0@lo:8:kworker/u8:3.0 Request processed in 152us (219us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099029.439606:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1352
00000100:00080000:3.0:1766099029.439656:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099029.439660:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099029.439664:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000242dd738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432902400:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766099029.886317:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099030.909865:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099031.937393:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099032.957700:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099033.886109:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099033.888731:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203264
00000100:00100000:3.0:1766099033.888797:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1500
00000100:00100000:3.0:1766099033.888803:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417203264:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099033.888891:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417203264:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 88us (2791us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099033.888899:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1500
00000100:00100000:2.0:1766099033.889225:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099033.889367:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099033.889434:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203392
00000100:00100000:0.0:1766099033.889466:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1353
00000100:00100000:0.0:1766099033.889471:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417203392:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099033.889592:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099033.889630:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417203392:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 158us (264us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099033.889636:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1353
00000100:00100000:3.0:1766099033.890562:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203328
00000100:00100000:3.0:1766099033.890651:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2207
00000100:00100000:3.0:1766099033.890656:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417203328:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099033.890728:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417203328:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 70us (1504us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099033.890735:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2207
00010000:00010000:0.0:1766099033.981908:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099034.557723:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:1.0:1766099034.557729:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766099034.557730:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099034.557732:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099034.557740:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3537
00000100:00080000:2.0:1766099034.557743:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:1.0:1766099034.557745:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432902592:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099034.557748:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099034.557768:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004039bd81 x1851885432902720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099034.557808:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432902784:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099034.557809:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099034.557838:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099034.557848:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007f24045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432902848:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099034.557854:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099034.557857:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432902656:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099034.557870:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099034.557874:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099034.557893:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432902720:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099034.557926:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099034.557929:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902784
00000100:00100000:2.0:1766099034.557950:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902656
00000100:00100000:1.0:1766099034.557966:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2208
00000100:00100000:2.0:1766099034.557969:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2209
00000100:00100000:1.0:1766099034.557971:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432902784:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099034.557972:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432902656:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099034.558011:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432902784:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (173us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099034.558018:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2208
00000100:00100000:1.0:1766099034.558041:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902848
00000100:00100000:1.0:1766099034.558060:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902720
00000100:00100000:1.0:1766099034.558074:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1014
00000100:00100000:1.0:1766099034.558078:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432902848:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099034.558107:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432902848:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (236us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099034.558112:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1014
00000100:00100000:1.0:1766099034.558122:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1354
00000100:00100000:1.0:1766099034.558126:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432902720:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099034.558127:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902592
00000100:00100000:0.0:1766099034.558148:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1501
00000100:00100000:0.0:1766099034.558151:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432902592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099034.558178:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432902592:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (370us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099034.558183:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1501
00010000:02020000:1.0:1766099034.558218:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099034.558246:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432902720:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (320us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099034.558252:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1354
00010000:00080000:1.0:1766099034.558283:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000007f24045 x1851885432902848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099041 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099034.558293:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007f24045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432902848:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099034.558301:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432902592:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099034.559928:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099034.559952:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099034.559957:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432902720:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099034.560161:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432902656:12345-0@lo:400:kworker/u8:2.0 Request processed in 2185us (2285us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099034.560169:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2209
00000100:00100000:3.0:1766099034.560388:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432902656:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099034.560400:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432902784:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099035.005733:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099036.029783:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099037.053810:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099038.077796:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099039.006896:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099039.006942:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099039.007020:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203520
00000100:00100000:1.0:1766099039.007033:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203456
00000100:00100000:1.0:1766099039.007088:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1502
00000100:00100000:1.0:1766099039.007095:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417203456:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099039.007103:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2210
00000100:00100000:2.0:1766099039.007109:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002bc5bf74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417203520:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099039.007180:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417203456:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 84us (302us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099039.007188:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1502
00000100:00100000:2.0:1766099039.007231:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002bc5bf74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417203520:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 121us (289us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099039.007238:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2210
00000100:00100000:2.0:1766099039.010376:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099039.010519:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203584
00000100:00100000:1.0:1766099039.010569:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1355
00000100:00100000:1.0:1766099039.010576:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417203584:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099039.010676:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099039.010733:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417203584:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 156us (363us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099039.010741:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1355
00010000:00010000:2.0:1766099039.104302:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099039.678743:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099039.678751:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099039.678755:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099039.678763:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3543
00000100:00080000:3.0:1766099039.678766:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:1.0:1766099039.678773:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766099039.678773:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099039.678792:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432902912:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.678794:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f7b8f472 x1851885432903040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099039.678843:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c7015c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432902976:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099039.678866:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099039.678893:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099039.678933:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099039.678939:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064d7ca73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432903104:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.678962:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099039.678993:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b8f472 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432903040:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099039.679018:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099039.679052:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902976
00000100:00100000:3.0:1766099039.679090:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903104
00000100:00100000:3.0:1766099039.679110:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2211
00000100:00100000:3.0:1766099039.679116:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c4f798bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432902976:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.679176:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c4f798bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432902976:12345-0@lo:400:kworker/u8:3.0 Request processed in 60us (284us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099039.679183:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2211
00000100:00100000:3.0:1766099039.679200:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2212
00000100:00100000:3.0:1766099039.679203:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003a6badc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432903104:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.679227:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003a6badc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432903104:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (265us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099039.679232:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2212
00000100:00100000:3.0:1766099039.679271:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064d7ca73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432903104:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.679281:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c7015c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432902976:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.679321:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903040
00000100:00100000:3.0:1766099039.679340:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1356
00000100:00100000:3.0:1766099039.679344:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3831caa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432903040:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099039.679436:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099039.679466:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3831caa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432903040:12345-0@lo:8:kworker/u8:3.0 Request processed in 121us (448us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099039.679472:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1356
00000100:00080000:3.0:1766099039.679516:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099039.679519:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099039.679523:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b8f472 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432903040:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099039.680787:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1307ce3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432903168:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099039.680830:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099039.680897:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432902912
00000100:00100000:1.0:1766099039.680931:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1503
00000100:00100000:1.0:1766099039.680935:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432902912:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099039.680986:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432902912:12345-0@lo:400:kworker/u8:3.0 Request processed in 50us (2122us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099039.680993:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1503
00000100:00100000:1.0:1766099039.681029:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432902912:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.684572:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903168
00000100:00100000:3.0:1766099039.684648:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1015
00000100:00100000:3.0:1766099039.684655:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fd582b8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432903168:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099039.684882:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fd582b8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432903168:12345-0@lo:400:kworker/u8:3.0 Request processed in 218us (4043us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099039.684890:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1015
00010000:00080000:1.0:1766099039.686321:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c1307ce3 x1851885432903168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099046 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099039.686340:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1307ce3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432903168:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099040.125815:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099041.149783:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099042.174071:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099043.197673:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099044.126429:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099044.126766:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099044.127352:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203648
00000100:00100000:2.0:1766099044.127428:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1504
00000100:00100000:2.0:1766099044.127435:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417203648:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099044.127554:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417203648:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 118us (1136us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099044.127564:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1504
00000100:00100000:0.0:1766099044.128547:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203712
00000100:00100000:0.0:1766099044.128593:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2213
00000100:00100000:0.0:1766099044.128598:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051f529aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417203712:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099044.128689:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051f529aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417203712:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (1924us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099044.128696:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2213
00000100:00100000:2.0:1766099044.130078:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099044.130294:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203776
00000100:00100000:3.0:1766099044.130333:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1357
00000100:00100000:3.0:1766099044.130340:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417203776:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099044.130467:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099044.130825:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417203776:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 483us (752us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099044.130834:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1357
00010000:00010000:2.0:1766099044.221835:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099044.798017:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099044.798025:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099044.798030:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099044.798039:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3548
00000100:00080000:1.0:1766099044.798044:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099044.798051:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099044.798076:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000af3b0884 x1851885432903360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099044.798128:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f0d53e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432903296:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798214:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099044.798247:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099044.798253:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5c3581c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432903424:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798276:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099044.798305:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af3b0884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432903360:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798334:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099044.798395:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903296
00000100:00100000:1.0:1766099044.798443:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903424
00000100:00100000:1.0:1766099044.798468:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2214
00000100:00100000:1.0:1766099044.798506:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000242dd738 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432903296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798568:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000242dd738 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432903296:12345-0@lo:400:kworker/u8:3.0 Request processed in 92us (354us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099044.798577:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2214
00000100:00100000:1.0:1766099044.798599:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2215
00000100:00100000:1.0:1766099044.798602:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000685e9d1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432903424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798628:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000685e9d1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432903424:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (351us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099044.798633:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2215
00000100:00100000:1.0:1766099044.798686:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5c3581c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432903424:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798698:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f0d53e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432903296:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.798729:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903360
00000100:00100000:1.0:1766099044.798749:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1358
00000100:00100000:1.0:1766099044.798753:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b4351f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432903360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099044.798873:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099044.798909:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b4351f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432903360:12345-0@lo:8:kworker/u8:3.0 Request processed in 155us (575us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099044.798916:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1358
00000100:00080000:1.0:1766099044.798940:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099044.798945:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099044.798948:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af3b0884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432903360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099044.799049:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000844e5d17 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432903232:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099044.799105:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099044.799138:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099044.799144:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3e3a280 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432903488:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099044.799168:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099044.799196:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903488
00000100:00100000:2.0:1766099044.799227:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1016
00000100:00100000:2.0:1766099044.799230:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432903488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099044.799265:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432903488:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (96us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099044.799278:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1016
00010000:00080000:2.0:1766099044.799309:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e3e3a280 x1851885432903488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099051 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099044.799319:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3e3a280 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432903488:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099044.799558:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903232
00000100:00100000:2.0:1766099044.800141:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1505
00000100:00100000:2.0:1766099044.800148:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432903232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099044.801747:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432903232:12345-0@lo:400:kworker/u8:3.0 Request processed in 1591us (2635us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099044.801764:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1505
00000100:00100000:0.0:1766099044.801887:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000844e5d17 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432903232:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099045.247451:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099046.270234:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099047.294342:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099048.317866:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099049.247034:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099049.247103:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099049.247123:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203904
00000100:00100000:0.0:1766099049.247200:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2216
00000100:00100000:0.0:1766099049.247209:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af3b0884 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417203904:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099049.248572:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203840
00000100:00100000:3.0:1766099049.249022:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1506
00000100:00100000:3.0:1766099049.249030:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417203840:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099049.249869:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af3b0884 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417203904:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2657us (2843us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099049.249882:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2216
00000100:00100000:2.0:1766099049.250516:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099049.251738:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417203968
00000100:00100000:0.0:1766099049.251785:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1359
00000100:00100000:0.0:1766099049.251792:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417203968:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099049.251909:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099049.251976:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417203968:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 183us (1462us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099049.251984:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1359
00000100:00100000:3.0:1766099049.252090:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417203840:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 3057us (4985us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099049.252100:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1506
00010000:00010000:0.0:1766099049.341986:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099049.917709:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000070350b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432903552:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099049.917757:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432903616:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099049.917774:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099049.917784:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099049.917789:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099049.917826:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3553
00000100:00080000:0.0:1766099049.917832:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:1.0:1766099049.917835:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766099049.917863:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099049.917898:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000081de4e5b x1851885432903680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099049.917922:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903616
00000100:00100000:1.0:1766099049.917986:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2217
00000100:00100000:0.0:1766099049.917992:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432903744:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099049.917998:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006f0d53e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432903616:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099049.918083:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099049.918090:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006f0d53e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432903616:12345-0@lo:400:kworker/u8:2.0 Request processed in 94us (258us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099049.918101:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2217
00000100:00100000:1.0:1766099049.918135:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903744
00000100:00100000:0.0:1766099049.918354:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432903680:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099049.918394:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099049.918442:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903680
00000100:00100000:0.0:1766099049.918579:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1360
00000100:00100000:0.0:1766099049.918585:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f198af5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432903680:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099049.918680:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099049.918728:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2218
00000100:00100000:1.0:1766099049.918737:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087b611a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432903744:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099049.918744:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f198af5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432903680:12345-0@lo:8:kworker/u8:2.0 Request processed in 153us (344us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099049.918756:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1360
00000100:00100000:2.0:1766099049.918817:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432903616:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099049.918828:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087b611a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432903744:12345-0@lo:400:kworker/u8:2.0 Request processed in 87us (746us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099049.918838:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2218
00000100:00080000:0.0:1766099049.918856:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099049.918863:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099049.918869:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432903680:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099049.918895:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099049.918923:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099049.918930:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5334aba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432903808:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099049.918940:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903552
00000100:00100000:2.0:1766099049.918959:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099049.918985:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1507
00000100:00100000:0.0:1766099049.918991:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432903552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099049.918996:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903808
00000100:00100000:1.0:1766099049.919008:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432903744:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099049.919036:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1017
00000100:00100000:2.0:1766099049.919043:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432903808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099049.919048:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432903552:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (153us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099049.919057:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1507
00000100:00100000:2.0:1766099049.919091:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432903808:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (132us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099049.919099:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1017
00000100:00100000:2.0:1766099049.919131:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000070350b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432903552:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766099049.919165:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f5334aba x1851885432903808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099056 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099049.919174:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5334aba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432903808:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099050.365956:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099051.389736:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099052.413729:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099053.439428:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099054.366563:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099054.366628:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099054.366722:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204032
00000100:00100000:3.0:1766099054.366798:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1508
00000100:00100000:3.0:1766099054.366806:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417204032:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099054.366912:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417204032:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 106us (284us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099054.366921:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1508
00000100:00100000:0.0:1766099054.367093:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204160
00000100:00100000:0.0:1766099054.367197:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1361
00000100:00100000:0.0:1766099054.367202:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5c3581c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417204160:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099054.367310:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099054.367992:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5c3581c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417204160:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 786us (1439us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099054.368001:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1361
00000100:00100000:1.0:1766099054.368871:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099054.369455:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204096
00000100:00100000:2.0:1766099054.369542:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2219
00000100:00100000:2.0:1766099054.369549:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000844e5d17 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417204096:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099054.369640:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000844e5d17 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417204096:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (773us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099054.369648:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2219
00010000:00010000:2.0:1766099054.462904:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099055.037864:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099055.037874:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099055.037895:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099055.037906:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3558
00000100:00080000:1.0:1766099055.037911:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099055.037918:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099055.037952:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004039bd81 x1851885432904000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099055.038011:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432904064:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038121:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099055.038159:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099055.038165:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432903936:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038189:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099055.038227:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904064
00000100:00100000:1.0:1766099055.038277:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903936
00000100:00100000:1.0:1766099055.038300:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2220
00000100:00100000:1.0:1766099055.038306:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432904064:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038375:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432904064:12345-0@lo:400:kworker/u8:3.0 Request processed in 67us (256us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099055.038382:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432904128:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038387:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2220
00000100:00100000:1.0:1766099055.038414:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2221
00000100:00100000:1.0:1766099055.038419:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432903936:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099055.038424:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099055.038447:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099055.038454:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3e3a280 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432903872:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038462:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432903936:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099055.038469:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2221
00000100:00100000:0.0:1766099055.038505:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099055.038542:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904000:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038552:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe333148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432903936:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038566:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3227a74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432904064:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099055.038570:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099055.038618:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432903872
00000100:00100000:1.0:1766099055.038633:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904128
00000100:00100000:0.0:1766099055.038653:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1509
00000100:00100000:0.0:1766099055.038659:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432903872:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038673:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904000
00000100:00100000:0.0:1766099055.038698:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432903872:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (193us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099055.038705:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1018
00000100:00100000:1.0:1766099055.038710:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081de4e5b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432904128:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099055.038719:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1509
00000100:00100000:1.0:1766099055.038751:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081de4e5b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432904128:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (327us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099055.038766:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1362
00000100:00100000:0.0:1766099055.038772:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6476c48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432904000:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099055.038774:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1018
00000100:00100000:1.0:1766099055.038817:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3e3a280 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432903872:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766099055.038827:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000aa4b7e05 x1851885432904128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099062 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099055.038836:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432904128:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766099055.038871:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099055.038922:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6476c48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432904000:12345-0@lo:8:kworker/u8:3.0 Request processed in 150us (352us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099055.038941:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1362
00000100:00080000:1.0:1766099055.040411:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099055.040422:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099055.040428:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004039bd81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904000:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099055.486357:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099056.510020:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099057.536229:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099058.559222:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099059.485292:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099059.485354:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099059.485395:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099059.485525:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204288
00000100:00100000:2.0:1766099059.485591:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2222
00000100:00100000:2.0:1766099059.485599:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3e3a280 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417204288:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099059.485746:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3e3a280 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417204288:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (412us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099059.485755:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2222
00000100:00100000:0.0:1766099059.491393:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204352
00000100:00100000:0.0:1766099059.491492:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1363
00000100:00100000:0.0:1766099059.491501:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417204352:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099059.491610:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099059.491713:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417204352:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 211us (6318us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099059.491722:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1363
00000100:00100000:3.0:1766099059.492448:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204224
00000100:00100000:3.0:1766099059.492627:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1510
00000100:00100000:3.0:1766099059.492669:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417204224:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099059.492766:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417204224:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 98us (7412us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099059.492774:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1510
00010000:00010000:2.0:1766099059.581682:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099060.158146:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099060.158154:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099060.158158:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099060.158187:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3563
00000100:00080000:2.0:1766099060.158192:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099060.158199:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099060.158230:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004584cb44 x1851885432904320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099060.158278:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432904256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099060.158368:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099060.158403:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099060.158409:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432904384:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099060.158432:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099060.158466:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904256
00000100:00100000:2.0:1766099060.158536:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904384
00000100:00100000:2.0:1766099060.158555:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2223
00000100:00100000:0.0:1766099060.158813:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432904192:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099060.158855:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099060.158878:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099060.158883:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432904448:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099060.158907:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099060.158936:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904448
00000100:00100000:0.0:1766099060.158964:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1019
00000100:00100000:0.0:1766099060.158970:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2c3700e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432904448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099060.159023:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2c3700e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432904448:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (115us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099060.159031:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1019
00010000:00080000:0.0:1766099060.159073:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e27d7f39 x1851885432904448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099067 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099060.159086:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e27d7f39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432904448:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099060.159205:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904320:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099060.159239:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099060.159350:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904192
00000100:00100000:3.0:1766099060.159379:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1511
00000100:00100000:3.0:1766099060.159383:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432904192:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099060.159424:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432904192:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (569us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099060.159430:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1511
00000100:00100000:3.0:1766099060.159471:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4dd01a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432904192:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099060.159691:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432904256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099060.159747:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432904256:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (1380us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099060.161646:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2224
00000100:00100000:0.0:1766099060.161662:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432904384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099060.161767:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002810083f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432904384:12345-0@lo:400:kworker/u8:2.0 Request processed in 109us (3334us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099060.161776:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2224
00000100:00100000:1.0:1766099060.162454:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904320
00000100:00100000:1.0:1766099060.162506:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1364
00000100:00100000:1.0:1766099060.162511:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432904320:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099060.163356:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099060.163441:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432904320:12345-0@lo:8:kworker/u8:2.0 Request processed in 928us (4200us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099060.163467:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1364
00000100:00080000:1.0:1766099060.163550:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099060.163556:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099060.163567:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904320:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099060.163645:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432904384:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099060.163659:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432904256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099060.163971:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2223
00010000:00010000:2.0:1766099060.605813:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099061.630772:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099062.653786:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099063.679325:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099064.605226:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099064.605282:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099064.605355:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204480
00000100:00100000:2.0:1766099064.605414:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2225
00000100:00100000:2.0:1766099064.605421:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7b8f472 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417204480:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099064.605533:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7b8f472 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417204480:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 111us (250us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099064.605540:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2225
00000100:00100000:0.0:1766099064.605645:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204416
00000100:00100000:0.0:1766099064.605685:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1512
00000100:00100000:0.0:1766099064.605698:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417204416:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099064.605751:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417204416:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 52us (532us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099064.605758:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1512
00000100:00100000:3.0:1766099064.609819:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099064.609897:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204544
00000100:00100000:1.0:1766099064.609944:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1365
00000100:00100000:1.0:1766099064.609951:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417204544:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099064.610909:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099064.610986:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417204544:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1034us (1175us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099064.610997:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1365
00010000:00010000:0.0:1766099064.701727:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099065.278817:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099065.278823:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099065.278827:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099065.278833:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3568
00000100:00080000:2.0:1766099065.278836:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099065.278841:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099065.278858:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000083116dde x1851885432904640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099065.278943:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432904512:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099065.279009:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099065.279047:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099065.279052:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062b9fb1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432904768:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099065.279077:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099065.279136:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432904704:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099065.279167:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099065.279191:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099065.279195:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b235acde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432904576:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099065.279214:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099065.279236:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083116dde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904640:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099065.279256:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099065.279285:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904768
00000100:00100000:1.0:1766099065.279320:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904640
00000100:00100000:1.0:1766099065.279333:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1020
00000100:00100000:1.0:1766099065.279346:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432904768:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099065.279387:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432904768:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (310us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099065.279393:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020
00000100:00100000:1.0:1766099065.279408:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1366
00000100:00100000:1.0:1766099065.279410:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432904640:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099065.279518:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099065.279544:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432904640:12345-0@lo:8:kworker/u8:2.0 Request processed in 133us (288us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099065.279549:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1366
00000100:00080000:1.0:1766099065.279582:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099065.279585:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099065.279588:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083116dde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904640:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099065.280137:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904704
00000100:00100000:3.0:1766099065.280166:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904576
00000100:00100000:3.0:1766099065.280179:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2226
00000100:00100000:3.0:1766099065.280182:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432904704:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099065.280219:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432904704:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (1052us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099065.280235:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2226
00000100:00100000:3.0:1766099065.280247:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2227
00000100:00100000:3.0:1766099065.280250:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432904576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099065.280272:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432904576:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (1058us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099065.280277:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2227
00000100:00100000:3.0:1766099065.280301:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b235acde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432904576:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099065.280309:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432904704:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766099065.280333:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000062b9fb1e x1851885432904768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099072 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099065.280342:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062b9fb1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432904768:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099065.280501:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904512
00000100:00100000:2.0:1766099065.280540:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1513
00000100:00100000:2.0:1766099065.280544:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432904512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099065.280584:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432904512:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (1577us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099065.280591:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1513
00000100:00100000:2.0:1766099065.280624:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432904512:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099065.726245:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099066.749703:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099067.773767:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099068.797941:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099069.725790:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099069.726000:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099069.726623:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099069.728203:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204608
00000100:00100000:2.0:1766099069.728268:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1514
00000100:00100000:2.0:1766099069.728275:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417204608:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099069.728344:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417204608:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 69us (1722us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099069.728351:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1514
00000100:00100000:3.0:1766099069.728684:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204736
00000100:00100000:3.0:1766099069.728709:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1367
00000100:00100000:3.0:1766099069.728713:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417204736:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099069.728813:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099069.728844:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417204736:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 130us (3063us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099069.728851:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1367
00000100:00100000:0.0:1766099069.729305:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204672
00000100:00100000:0.0:1766099069.729341:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2228
00000100:00100000:0.0:1766099069.729346:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417204672:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099069.729379:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000affb1b2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417204672:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 33us (3381us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099069.729386:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2228
00010000:00010000:2.0:1766099069.822921:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099070.401953:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099070.401960:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099070.401964:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099070.401972:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3573
00000100:00080000:2.0:1766099070.401975:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099070.401996:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099070.402018:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006c78fed0 x1851885432904960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099070.402081:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000075781527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432904896:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402147:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099070.402181:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099070.402185:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a77be11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432905024:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402206:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099070.402244:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904896
00000100:00100000:2.0:1766099070.402279:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905024
00000100:00100000:2.0:1766099070.402296:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2229
00000100:00100000:2.0:1766099070.402302:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000849b0c69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432904896:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402346:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000849b0c69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432904896:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (200us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099070.402353:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2229
00000100:00100000:2.0:1766099070.402369:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2230
00000100:00100000:2.0:1766099070.402373:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d6d3cb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432905024:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402399:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d6d3cb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432905024:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (193us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099070.402404:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2230
00000100:00100000:2.0:1766099070.402439:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a77be11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432905024:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402465:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000075781527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432904896:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402526:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008857350 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432905088:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402552:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099070.402570:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099070.402574:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432904832:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402597:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099070.402621:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905088
00000100:00100000:2.0:1766099070.402643:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1021
00000100:00100000:2.0:1766099070.402658:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075781527 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432905088:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402692:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075781527 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432905088:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (140us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099070.402698:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1021
00010000:00080000:2.0:1766099070.402724:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000008857350 x1851885432905088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099077 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099070.402732:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008857350 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432905088:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402750:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904832
00000100:00100000:2.0:1766099070.402771:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1515
00000100:00100000:2.0:1766099070.402775:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432904832:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402802:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432904832:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (204us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099070.402808:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1515
00000100:00100000:2.0:1766099070.402835:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432904832:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402862:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006c78fed0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904960:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099070.402893:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099070.402917:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432904960
00000100:00100000:2.0:1766099070.402933:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1368
00000100:00100000:2.0:1766099070.402937:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432904960:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099070.403027:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099070.403057:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432904960:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (164us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099070.403063:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1368
00000100:00080000:2.0:1766099070.403083:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099070.403086:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099070.403089:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006c78fed0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432904960:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099070.845783:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099071.869787:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099072.895671:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099073.919160:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099074.846103:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099074.846171:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099074.846777:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204800
00000100:00100000:3.0:1766099074.847875:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099074.847934:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204928
00000100:00100000:3.0:1766099074.847972:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1369
00000100:00100000:3.0:1766099074.847980:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064850b98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417204928:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766099074.848034:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1516
00000100:00100000:0.0:1766099074.848042:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417204800:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00010000:02020000:3.0:1766099074.848085:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099074.848156:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417204800:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 114us (1984us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099074.848165:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1516
00000100:00100000:2.0:1766099074.848212:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204864
00000100:00100000:2.0:1766099074.848286:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2231
00000100:00100000:2.0:1766099074.848294:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417204864:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099074.850639:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064850b98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417204928:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 2645us (2756us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099074.850659:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1369
00000100:00100000:2.0:1766099074.851425:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a6b62b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417204864:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 3127us (5330us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099074.851439:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2231
00010000:00010000:2.0:1766099074.942643:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099075.521794:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099075.521803:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099075.521807:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099075.521817:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3578
00000100:00080000:0.0:1766099075.521821:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099075.521828:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099075.521852:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d7a3c8dd x1851885432905280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099075.521943:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432905408:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522021:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099075.522071:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099075.522075:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008077429a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432905152:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522112:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099075.522179:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dffd2c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432905344:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522208:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099075.522226:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099075.522231:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432905216:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522251:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099075.522274:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432905280:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522296:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099075.522338:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905344
00000100:00100000:2.0:1766099075.522388:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905216
00000100:00100000:2.0:1766099075.522409:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2232
00000100:00100000:2.0:1766099075.522415:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008857350 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432905344:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522519:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008857350 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432905344:12345-0@lo:400:kworker/u8:2.0 Request processed in 101us (308us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099075.522527:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2232
00000100:00100000:2.0:1766099075.522547:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2233
00000100:00100000:2.0:1766099075.522552:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a77be11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432905216:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099075.522575:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a77be11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432905216:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (324us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099075.522581:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905408
00000100:00100000:2.0:1766099075.522582:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2233
00000100:00100000:2.0:1766099075.522650:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432905216:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099075.522664:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905280
00000100:00100000:2.0:1766099075.522665:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dffd2c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432905344:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099075.522704:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1022
00000100:00100000:0.0:1766099075.522712:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006c78fed0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432905408:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099075.522823:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006c78fed0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432905408:12345-0@lo:400:kworker/u8:2.0 Request processed in 110us (801us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099075.522834:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1022
00010000:00080000:2.0:1766099075.522849:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006b09b6fb x1851885432905408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099082 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099075.522860:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1370
00000100:00100000:2.0:1766099075.522863:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432905408:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099075.522870:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9c2be5b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432905280:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099075.522965:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099075.523011:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9c2be5b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432905280:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (715us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099075.523021:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1370
00000100:00100000:1.0:1766099075.523185:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905152
00000100:00100000:1.0:1766099075.523234:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1517
00000100:00100000:1.0:1766099075.523239:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432905152:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099075.523291:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432905152:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (1182us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099075.523297:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1517
00000100:00100000:1.0:1766099075.523352:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008077429a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432905152:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099075.523708:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099075.523713:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099075.523716:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432905280:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099075.967590:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099076.989842:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099078.015455:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099079.038527:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099079.969822:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099079.970884:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099079.970972:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205056
00000100:00100000:1.0:1766099079.970996:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099079.971039:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2234
00000100:00100000:2.0:1766099079.971047:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417205056:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099079.971103:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417204992
00000100:00100000:0.0:1766099079.971139:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1518
00000100:00100000:0.0:1766099079.971143:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417204992:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099079.971146:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025c2fc30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417205056:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 100us (1339us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099079.971153:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2234
00000100:00100000:0.0:1766099079.971171:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417204992:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 27us (292us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099079.971177:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1518
00000100:00100000:3.0:1766099079.971430:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205120
00000100:00100000:3.0:1766099079.971471:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1371
00000100:00100000:3.0:1766099079.972067:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205120:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766099079.972168:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099079.972231:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205120:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 754us (1234us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099079.972239:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1371
00010000:00010000:0.0:1766099080.061920:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099080.637773:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099080.637781:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:3.0:1766099080.637785:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000179e8d06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432905472:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099080.637786:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099080.637797:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3583
00000100:00100000:1.0:1766099080.637797:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057c7afc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432905536:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099080.637800:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099080.637809:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099080.637840:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000914f0967 x1851885432905600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099080.637889:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099080.637889:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099080.637930:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099080.637930:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099080.637941:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008077429a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432905728:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099080.637942:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432905664:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099080.637943:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905472
00000100:00100000:1.0:1766099080.637979:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099080.637988:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099080.637994:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1519
00000100:00100000:0.0:1766099080.638002:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432905472:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099080.638017:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905536
00000100:00100000:3.0:1766099080.638025:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905728
00000100:00100000:1.0:1766099080.638054:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905664
00000100:00100000:3.0:1766099080.638063:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1023
00000100:00100000:3.0:1766099080.638069:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432905728:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099080.638075:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432905472:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (190us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099080.638083:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2235
00000100:00100000:0.0:1766099080.638087:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1519
00000100:00100000:1.0:1766099080.638092:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432905536:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099080.638109:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432905728:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (122us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099080.638116:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1023
00000100:00100000:1.0:1766099080.638138:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432905536:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (254us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099080.638145:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2235
00000100:00100000:1.0:1766099080.638162:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2236
00000100:00100000:1.0:1766099080.638166:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432905664:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099080.638189:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432905664:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (210us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099080.638195:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2236
00000100:00100000:0.0:1766099080.638216:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432905664:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766099080.638217:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008077429a x1851885432905728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099087 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099080.638231:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008077429a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432905728:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099080.638238:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057c7afc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432905536:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099080.638264:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000179e8d06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432905472:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099080.638536:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000914f0967 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432905600:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099080.638583:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099080.638633:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905600
00000100:00100000:3.0:1766099080.638671:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1372
00000100:00100000:3.0:1766099080.638676:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000220e4534 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432905600:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099080.638780:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099080.638825:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000220e4534 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432905600:12345-0@lo:8:kworker/u8:2.0 Request processed in 148us (243us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099080.638832:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1372
00000100:00080000:3.0:1766099080.638863:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099080.638868:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099080.638873:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000914f0967 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432905600:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099081.086672:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099082.121977:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099083.134538:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099084.157823:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099085.102574:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099085.102699:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205184
00000100:00100000:2.0:1766099085.102755:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1520
00000100:00100000:2.0:1766099085.102761:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417205184:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099085.102837:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417205184:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 74us (271us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099085.102845:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1520
00000100:00100000:0.0:1766099085.104194:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099085.104234:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099085.104312:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205248
00000100:00100000:2.0:1766099085.104361:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2237
00000100:00100000:2.0:1766099085.104367:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057c7afc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417205248:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099085.104449:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057c7afc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417205248:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 81us (259us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099085.104456:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2237
00000100:00100000:3.0:1766099085.104986:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205312
00000100:00100000:3.0:1766099085.105019:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1373
00000100:00100000:3.0:1766099085.105024:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7a3c8dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205312:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099085.105128:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099085.108200:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7a3c8dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205312:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 3166us (3957us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099085.108216:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1373
00010000:00010000:2.0:1766099085.181747:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099085.757823:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099085.757831:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099085.757834:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099085.757843:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3589
00000100:00080000:0.0:1766099085.757846:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099085.757852:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099085.757871:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000081a397a0 x1851885432905920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099085.757916:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432905984:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099085.757982:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099085.758009:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099085.758013:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dffd2c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432905856:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099085.758052:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099085.758088:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905984
00000100:00100000:0.0:1766099085.758130:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905856
00000100:00100000:0.0:1766099085.758148:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2238
00000100:00100000:0.0:1766099085.758154:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6476c48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432905984:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099085.758201:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6476c48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432905984:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (220us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099085.758209:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2238
00000100:00100000:0.0:1766099085.758226:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2239
00000100:00100000:0.0:1766099085.758230:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2c3700e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432905856:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099085.758256:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2c3700e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432905856:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (204us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099085.758279:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2239
00000100:00100000:0.0:1766099085.758322:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dffd2c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432905856:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099085.758331:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432905984:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099085.758427:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432906048:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099085.758462:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099085.758511:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099085.758517:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432905792:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099085.758566:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099085.758600:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905792
00000100:00100000:3.0:1766099085.758602:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432905920:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099085.758631:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1521
00000100:00100000:1.0:1766099085.758642:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432905792:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099085.758642:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099085.758678:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906048
00000100:00100000:1.0:1766099085.758688:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432905792:12345-0@lo:400:kworker/u8:3.0 Request processed in 50us (122us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099085.758696:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1521
00000100:00100000:3.0:1766099085.758705:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432905920
00000100:00100000:3.0:1766099085.758722:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1024
00000100:00100000:3.0:1766099085.758725:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432906048:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099085.758740:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1374
00000100:00100000:1.0:1766099085.758746:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432905920:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099085.758764:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432906048:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (301us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099085.758771:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1024
00010000:02020000:1.0:1766099085.758834:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099085.758876:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432905920:12345-0@lo:8:kworker/u8:3.0 Request processed in 129us (241us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099085.758882:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1374
00000100:00100000:1.0:1766099085.758907:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432905792:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766099085.758916:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000081de4e5b x1851885432906048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099092 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099085.758925:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432906048:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099085.759086:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099085.759090:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099085.759094:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432905920:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099086.205687:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099087.229772:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099088.253786:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099089.278924:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099090.204624:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099090.204827:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205504
00000100:00100000:2.0:1766099090.204900:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1375
00000100:00100000:2.0:1766099090.204907:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f198af5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417205504:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766099090.204947:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099090.205004:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205440
00010000:02020000:2.0:1766099090.205005:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099090.205059:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2240
00000100:00100000:1.0:1766099090.205066:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dffd2c34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417205440:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099090.205078:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f198af5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417205504:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 163us (454us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099090.205085:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099090.205095:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1375
00000100:00100000:1.0:1766099090.205112:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dffd2c34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417205440:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (172us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099090.205118:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2240
00000100:00100000:2.0:1766099090.205157:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205376
00000100:00100000:2.0:1766099090.205199:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1522
00000100:00100000:2.0:1766099090.205204:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417205376:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099090.205256:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417205376:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 52us (186us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099090.205263:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1522
00010000:00010000:0.0:1766099090.302105:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099090.878181:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099090.878192:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099090.878196:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099090.878206:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3594
00000100:00080000:2.0:1766099090.878210:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099090.878217:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099090.878264:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000069aac5df x1851885432906240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099090.878322:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069aac5df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432906240:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099090.878421:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099090.878516:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906240
00000100:00100000:2.0:1766099090.878570:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1376
00000100:00100000:2.0:1766099090.878578:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008e61d6e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432906240:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099090.878683:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099090.878733:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008e61d6e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432906240:12345-0@lo:8:kworker/u8:2.0 Request processed in 155us (313us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099090.878741:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1376
00000100:00080000:2.0:1766099090.878783:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099090.878787:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099090.878791:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069aac5df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432906240:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099090.878843:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432906368:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099090.878873:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099090.878940:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080dc22a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432906112:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099090.879056:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ac37e31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432906176:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099090.879583:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906368
00000100:00100000:1.0:1766099090.879641:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1025
00000100:00100000:1.0:1766099090.879647:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069aac5df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432906368:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099090.883782:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099090.883859:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906112
00000100:00100000:3.0:1766099090.883911:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1523
00000100:00100000:3.0:1766099090.883919:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432906112:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099090.883989:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432906112:12345-0@lo:400:kworker/u8:2.0 Request processed in 70us (216us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099090.883997:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1523
00000100:00100000:3.0:1766099090.884054:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080dc22a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432906112:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099090.884097:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432906304:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099090.884132:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099090.884161:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906304
00000100:00100000:3.0:1766099090.884189:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2241
00000100:00100000:3.0:1766099090.884192:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000305829f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432906304:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099090.884223:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000305829f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432906304:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (91us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099090.884228:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2241
00000100:00100000:3.0:1766099090.884250:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432906304:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099090.884782:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099090.884860:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906176
00000100:00100000:3.0:1766099090.884890:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2242
00000100:00100000:3.0:1766099090.884893:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5334aba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432906176:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099090.884928:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5334aba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432906176:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (148us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099090.884933:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2242
00000100:00100000:0.0:1766099090.885053:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ac37e31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432906176:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099090.885537:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069aac5df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432906368:12345-0@lo:400:kworker/u8:2.0 Request processed in 5882us (6656us total) trans 0 rc 0/0
00010000:00080000:3.0:1766099090.885546:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c9d7b580 x1851885432906368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099097 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099090.885561:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1025
00000100:00100000:3.0:1766099090.885567:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432906368:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099091.325756:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099092.352605:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099093.374157:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099094.403333:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099095.325768:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099095.325831:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099095.325867:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099095.325976:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205568
00000100:00100000:1.0:1766099095.326044:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1524
00000100:00100000:1.0:1766099095.326051:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417205568:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099095.326138:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417205568:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 87us (308us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099095.326146:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1524
00000100:00100000:2.0:1766099095.327066:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205696
00000100:00100000:2.0:1766099095.327123:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1377
00000100:00100000:2.0:1766099095.327129:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad1e01cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205696:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766099095.327231:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099095.327294:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad1e01cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205696:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 164us (1426us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099095.327697:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205632
00000100:00100000:3.0:1766099095.327785:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2243
00000100:00100000:3.0:1766099095.327791:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000070350b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417205632:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099095.327845:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000070350b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417205632:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 54us (2093us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099095.327853:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2243
00000100:00100000:2.0:1766099095.328121:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1377
00010000:00010000:0.0:1766099095.422098:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099095.997780:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099095.997791:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099095.997796:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099095.997807:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3599
00000100:00080000:1.0:1766099095.997812:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099095.997819:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099095.997852:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b0591111 x1851885432906560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099095.997998:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432906560:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099095.998097:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099095.998164:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432906496:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099095.998195:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099095.998217:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099095.998223:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432906624:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099095.998247:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099095.998288:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906496
00000100:00100000:2.0:1766099095.998335:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906624
00000100:00100000:2.0:1766099095.998355:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2244
00000100:00100000:2.0:1766099095.998361:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014c9a630 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432906496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099095.998416:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014c9a630 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432906496:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (221us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099095.998424:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2244
00000100:00100000:2.0:1766099095.998441:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2245
00000100:00100000:2.0:1766099095.998445:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006972e75c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432906624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099095.998527:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006972e75c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432906624:12345-0@lo:400:kworker/u8:3.0 Request processed in 81us (279us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099095.998534:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2245
00000100:00100000:2.0:1766099095.998578:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432906624:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099095.998589:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432906496:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099095.998673:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906560
00000100:00100000:1.0:1766099095.998705:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1378
00000100:00100000:1.0:1766099095.998710:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c637466 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432906560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099095.998822:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099095.998865:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c637466 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432906560:12345-0@lo:8:kworker/u8:3.0 Request processed in 154us (770us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099095.998892:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1378
00000100:00080000:1.0:1766099095.998923:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099095.998928:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099095.998932:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432906560:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099095.999528:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432906688:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099095.999579:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099095.999614:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099095.999622:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055b9c8f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432906432:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099095.999666:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099095.999784:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906688
00000100:00100000:2.0:1766099095.999832:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1026
00000100:00100000:2.0:1766099095.999849:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432906688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099095.999901:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432906688:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (322us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099095.999910:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1026
00000100:00100000:1.0:1766099096.000013:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906432
00000100:00100000:1.0:1766099096.000056:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1525
00000100:00100000:1.0:1766099096.000060:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432906432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099096.000097:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432906432:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (431us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099096.000103:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1525
00000100:00100000:3.0:1766099096.000470:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055b9c8f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432906432:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099096.000521:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001629b413 x1851885432906688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099102 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099096.000532:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432906688:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099096.445861:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099097.469871:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099098.493644:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099099.519966:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099100.447331:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099100.448545:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205824
00000100:00100000:0.0:1766099100.448633:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2246
00000100:00100000:0.0:1766099100.448640:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417205824:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099100.448785:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417205824:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 143us (1467us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099100.448793:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2246
00000100:00100000:1.0:1766099100.453460:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099100.453549:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099100.453775:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205760
00000100:00100000:1.0:1766099100.454376:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1526
00000100:00100000:1.0:1766099100.454387:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417205760:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099100.454537:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417205760:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 150us (988us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099100.454547:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1526
00000100:00100000:0.0:1766099100.454742:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205888
00000100:00100000:0.0:1766099100.454787:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1379
00000100:00100000:0.0:1766099100.454792:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205888:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766099100.454897:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099100.454937:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417205888:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 144us (1488us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099100.454944:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1379
00010000:00010000:2.0:1766099100.542496:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099101.118566:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099101.118577:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099101.118582:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099101.118594:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3604
00000100:00080000:2.0:1766099101.118599:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099101.118607:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099101.118636:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bfede6c4 x1851885432906880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099101.118667:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a949ae5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432906752:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099101.118693:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432907008:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099101.118933:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432906880:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099101.119003:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432906816:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099101.119216:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099101.119265:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432906944:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099101.119298:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099101.119299:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099101.119361:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906944
00000100:00100000:2.0:1766099101.119361:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906752
00000100:00100000:1.0:1766099101.119428:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2247
00000100:00100000:2.0:1766099101.119429:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1527
00000100:00100000:2.0:1766099101.119436:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432906752:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099101.119453:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba35ba7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432906944:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099101.119522:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432906752:12345-0@lo:400:kworker/u8:3.0 Request processed in 84us (305us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099101.119530:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1527
00000100:00100000:1.0:1766099101.119534:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba35ba7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432906944:12345-0@lo:400:kworker/u8:3.0 Request processed in 80us (237us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099101.119542:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2247
00000100:00100000:2.0:1766099101.119578:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907008
00000100:00100000:1.0:1766099101.119585:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432906944:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099101.119611:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1027
00000100:00100000:2.0:1766099101.119617:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ac37e31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432907008:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099101.119649:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a949ae5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432906752:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099101.119672:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ac37e31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432907008:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (378us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099101.119680:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1027
00010000:00080000:2.0:1766099101.119707:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c9d7b580 x1851885432907008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099108 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099101.119717:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432907008:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099101.119779:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099101.119829:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906880
00000100:00100000:2.0:1766099101.119865:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1380
00000100:00100000:3.0:1766099101.119865:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099101.119873:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049dffb54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432906880:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099101.119991:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099101.120029:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049dffb54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432906880:12345-0@lo:8:kworker/u8:3.0 Request processed in 157us (251us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099101.120037:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1380
00000100:00080000:3.0:1766099101.120052:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099101.120058:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099101.120062:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432906880:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099101.120130:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432906816
00000100:00100000:0.0:1766099101.120166:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2248
00000100:00100000:0.0:1766099101.120170:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3831caa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432906816:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099101.120218:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3831caa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432906816:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (358us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099101.120224:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2248
00000100:00100000:3.0:1766099101.120291:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432906816:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099101.565800:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099102.589766:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099103.613880:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099104.637761:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099105.565331:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099105.565406:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099105.565446:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099105.565587:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206080
00000100:00100000:2.0:1766099105.565645:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1381
00000100:00100000:2.0:1766099105.565654:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417206080:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766099105.565754:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099105.565969:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206016
00000100:00100000:0.0:1766099105.566531:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2249
00000100:00100000:0.0:1766099105.566538:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066675ebb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417206016:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099105.566644:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066675ebb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417206016:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 105us (1324us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099105.566652:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2249
00000100:00100000:3.0:1766099105.566741:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417205952
00000100:00100000:3.0:1766099105.566772:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1528
00000100:00100000:3.0:1766099105.566776:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417205952:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099105.566803:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417205952:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 26us (1398us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099105.566809:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1528
00000100:00100000:2.0:1766099105.592081:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417206080:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 26418us (26626us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099105.592101:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1381
00010000:00010000:0.0:1766099105.663786:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099106.237697:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099106.237707:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099106.237711:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099106.237741:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3609
00000100:00080000:2.0:1766099106.237746:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099106.237753:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099106.237782:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000080dc22a4 x1851885432907200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099106.237921:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432907072:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238015:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099106.238059:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099106.238064:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432907328:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238093:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099106.238130:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000075781527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432907264:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238162:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099106.238179:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099106.238183:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432907136:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238206:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099106.238250:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907264
00000100:00100000:1.0:1766099106.238309:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907136
00000100:00100000:1.0:1766099106.238332:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2250
00000100:00100000:1.0:1766099106.238338:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fd582b8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432907264:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238406:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fd582b8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432907264:12345-0@lo:400:kworker/u8:3.0 Request processed in 66us (243us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099106.238421:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2250
00000100:00100000:1.0:1766099106.238449:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2251
00000100:00100000:1.0:1766099106.238452:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b4351f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432907136:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238517:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b4351f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432907136:12345-0@lo:400:kworker/u8:3.0 Request processed in 64us (311us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099106.238524:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2251
00000100:00100000:1.0:1766099106.238573:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432907136:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099106.238583:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000075781527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432907264:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099106.238677:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080dc22a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432907200:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099106.238712:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099106.239915:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907072
00000100:00100000:2.0:1766099106.239972:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1529
00000100:00100000:2.0:1766099106.239977:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432907072:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099106.240051:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432907072:12345-0@lo:400:kworker/u8:3.0 Request processed in 74us (2038us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099106.240059:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1529
00000100:00100000:2.0:1766099106.241667:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907328
00000100:00100000:2.0:1766099106.241746:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907200
00000100:00100000:2.0:1766099106.241780:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1028
00000100:00100000:2.0:1766099106.241786:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432907328:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099106.241885:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432907328:12345-0@lo:400:kworker/u8:3.0 Request processed in 99us (3792us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099106.241892:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1028
00000100:00100000:2.0:1766099106.241909:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1382
00000100:00100000:2.0:1766099106.241913:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432907200:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099106.242010:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099106.242046:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432907200:12345-0@lo:8:kworker/u8:3.0 Request processed in 132us (3334us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099106.242052:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1382
00010000:00080000:2.0:1766099106.242135:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c2880771 x1851885432907328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099113 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099106.242178:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432907328:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099106.242198:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432907072:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099106.242331:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099106.242337:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099106.242341:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080dc22a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432907200:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766099106.685871:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099107.710321:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099108.733747:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099109.757719:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099110.688537:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099110.688605:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099110.689411:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206272
00000100:00100000:0.0:1766099110.689514:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1383
00000100:00100000:0.0:1766099110.689522:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417206272:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099110.689636:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099110.689717:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417206272:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 194us (1193us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099110.689725:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1383
00000100:00100000:3.0:1766099110.689874:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099110.690016:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206208
00000100:00100000:1.0:1766099110.690064:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2252
00000100:00100000:1.0:1766099110.690070:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ec0c7a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417206208:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099110.691947:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206144
00000100:00100000:2.0:1766099110.692018:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1530
00000100:00100000:2.0:1766099110.692025:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206144:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099110.692884:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ec0c7a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417206208:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2811us (3010us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099110.692896:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2252
00000100:00100000:2.0:1766099110.693146:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206144:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1120us (4541us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099110.693154:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1530
00010000:00010000:0.0:1766099110.782863:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099111.357823:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099111.357833:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099111.357837:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099111.357847:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3614
00000100:00080000:1.0:1766099111.357851:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099111.357857:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099111.357880:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006dd103c6 x1851885432907520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099111.357927:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432907584:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358041:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099111.358082:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099111.358087:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432907456:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358109:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099111.358145:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907584
00000100:00100000:1.0:1766099111.358188:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907456
00000100:00100000:1.0:1766099111.358205:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2253
00000100:00100000:1.0:1766099111.358211:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432907584:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358257:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432907584:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (217us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099111.358264:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2253
00000100:00100000:1.0:1766099111.358282:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2254
00000100:00100000:1.0:1766099111.358285:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432907456:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358302:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432907456:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (193us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099111.358307:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2254
00000100:00100000:1.0:1766099111.358386:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432907456:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358396:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432907584:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099111.358520:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432907648:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099111.358554:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099111.358584:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099111.358589:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5c3581c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432907392:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099111.358614:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099111.358639:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907392
00000100:00100000:2.0:1766099111.358662:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1531
00000100:00100000:2.0:1766099111.358665:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432907392:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099111.358693:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432907392:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (79us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099111.358699:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1531
00000100:00100000:2.0:1766099111.358727:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5c3581c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432907392:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358784:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907648
00000100:00100000:1.0:1766099111.358814:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1029
00000100:00100000:1.0:1766099111.358818:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432907648:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099111.358846:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432907648:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (292us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099111.358851:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1029
00010000:00080000:2.0:1766099111.358910:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000aa4b7e05 x1851885432907648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099118 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099111.358928:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432907648:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099111.363729:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006dd103c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432907520:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099111.363830:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099111.364104:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907520
00000100:00100000:2.0:1766099111.364163:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1384
00000100:00100000:2.0:1766099111.364170:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432907520:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099111.364273:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099111.364351:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432907520:12345-0@lo:8:kworker/u8:2.0 Request processed in 180us (521us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099111.364358:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1384
00000100:00080000:3.0:1766099111.364723:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099111.364747:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099111.364751:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006dd103c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432907520:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099111.806118:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099112.830547:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099113.853834:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099114.878357:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099115.805637:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099115.805810:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099115.805854:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099115.806000:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206336
00000100:00100000:3.0:1766099115.806021:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206464
00000100:00100000:3.0:1766099115.806082:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1385
00000100:00100000:1.0:1766099115.806083:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1532
00000100:00100000:1.0:1766099115.806096:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206336:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099115.806096:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002df41de4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417206464:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:1.0:1766099115.806194:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206336:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 102us (384us total) trans 0 rc 0/0
00010000:02020000:3.0:1766099115.806201:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099115.806206:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1532
00000100:00100000:3.0:1766099115.806241:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002df41de4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417206464:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 151us (615us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099115.806249:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1385
00000100:00100000:2.0:1766099115.808962:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206400
00000100:00100000:2.0:1766099115.809059:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2255
00000100:00100000:2.0:1766099115.809067:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097631d91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417206400:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099115.809173:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097631d91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417206400:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 105us (3318us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099115.809182:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2255
00010000:00010000:2.0:1766099115.902058:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099116.479397:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099116.479406:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099116.479411:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099116.479422:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3619
00000100:00080000:0.0:1766099116.479425:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099116.479433:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099116.479513:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000081de4e5b x1851885432907840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099116.480177:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432907904:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.480258:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099116.480292:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099116.480297:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432907776:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.480332:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099116.480378:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0765a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432907712:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.480386:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907904
00000100:00100000:2.0:1766099116.480428:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099116.480434:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907776
00000100:00100000:2.0:1766099116.480455:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099116.480461:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2256
00000100:00100000:2.0:1766099116.480464:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000914f0967 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432907968:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.480470:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432907904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099116.480514:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099116.480551:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2257
00000100:00100000:1.0:1766099116.480555:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432907904:12345-0@lo:400:kworker/u8:2.0 Request processed in 84us (296us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099116.480560:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432907776:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.480568:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2256
00000100:00100000:2.0:1766099116.480599:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432907776:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (266us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099116.480607:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2257
00000100:00100000:1.0:1766099116.480610:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907712
00000100:00100000:2.0:1766099116.480653:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907968
00000100:00100000:2.0:1766099116.480683:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1030
00000100:00100000:2.0:1766099116.480686:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432907968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099116.480717:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432907968:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (204us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099116.480722:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1030
00010000:00080000:2.0:1766099116.480755:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000914f0967 x1851885432907968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099123 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099116.480770:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000914f0967 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432907968:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099116.481012:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432907840:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099116.481038:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099116.481072:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432907840
00000100:00100000:2.0:1766099116.481090:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1386
00000100:00100000:2.0:1766099116.481094:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062b9fb1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432907840:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099116.481242:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099116.481272:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062b9fb1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432907840:12345-0@lo:8:kworker/u8:2.0 Request processed in 178us (234us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099116.481279:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1386
00000100:00080000:2.0:1766099116.481317:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099116.481322:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099116.481325:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432907840:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099116.485038:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1533
00000100:00100000:1.0:1766099116.485053:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432907712:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.485554:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432907712:12345-0@lo:400:kworker/u8:2.0 Request processed in 501us (5126us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099116.485563:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1533
00000100:00100000:1.0:1766099116.485653:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432907776:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099116.485666:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432907904:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099116.488873:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0765a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432907712:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099116.926887:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099117.950032:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099118.973945:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099120.000956:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099120.928142:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099120.928664:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099120.928916:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206592
00000100:00100000:2.0:1766099120.929001:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2258
00000100:00100000:2.0:1766099120.929008:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000179e8d06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417206592:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099120.929106:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000179e8d06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417206592:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 97us (974us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099120.929114:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2258
00000100:00100000:1.0:1766099120.933552:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206528
00000100:00100000:1.0:1766099120.933642:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1534
00000100:00100000:1.0:1766099120.933650:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417206528:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099120.933753:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417206528:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 103us (5090us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099120.933760:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1534
00000100:00100000:2.0:1766099120.935191:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099120.935542:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206656
00000100:00100000:3.0:1766099120.935750:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1387
00000100:00100000:3.0:1766099120.935757:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417206656:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766099120.935863:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099120.944698:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417206656:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 8929us (9502us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099120.944718:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1387
00010000:00010000:0.0:1766099121.021856:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099121.598403:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099121.598413:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099121.598416:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099121.598425:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3624
00000100:00080000:0.0:1766099121.598428:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099121.598434:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099121.598456:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006b09b6fb x1851885432908160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099121.598556:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432908224:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.598675:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008077429a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432908032:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.598877:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099121.598908:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099121.598912:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432908288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.598933:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099121.598967:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908032
00000100:00100000:3.0:1766099121.599001:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1535
00000100:00100000:3.0:1766099121.599005:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432908032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599058:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432908032:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (182us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099121.599065:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1535
00000100:00100000:3.0:1766099121.599092:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908288
00000100:00100000:3.0:1766099121.599110:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1031
00000100:00100000:3.0:1766099121.599113:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432908288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599135:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432908288:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (203us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099121.599141:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1031
00010000:00080000:3.0:1766099121.599171:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000081de4e5b x1851885432908288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099128 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099121.599182:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081de4e5b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432908288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599189:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008077429a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432908032:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599217:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0765a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432908096:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599237:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099121.599282:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432908160:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599303:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099121.599332:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908160
00000100:00100000:3.0:1766099121.599353:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1388
00000100:00100000:3.0:1766099121.599356:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432908160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099121.599443:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099121.599473:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432908160:12345-0@lo:8:kworker/u8:3.0 Request processed in 115us (170us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099121.599502:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1388
00000100:00080000:3.0:1766099121.599529:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099121.599533:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099121.599536:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b09b6fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432908160:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599567:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908096
00000100:00100000:3.0:1766099121.599590:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2259
00000100:00100000:3.0:1766099121.599594:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432908096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599635:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432908096:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (397us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099121.599641:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2259
00000100:00100000:3.0:1766099121.599668:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0765a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432908096:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099121.599751:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099121.599845:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908224
00000100:00100000:3.0:1766099121.599873:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2260
00000100:00100000:3.0:1766099121.599876:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000914f0967 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432908224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099121.599904:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000914f0967 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432908224:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (154us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099121.599916:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2260
00000100:00100000:0.0:1766099121.599979:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081a397a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432908224:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099122.046757:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099123.069755:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099124.095353:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099125.117724:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099126.047904:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099126.048968:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099126.049004:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099126.049063:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206720
00000100:00100000:1.0:1766099126.049079:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206848
00000100:00100000:3.0:1766099126.049136:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1536
00000100:00100000:3.0:1766099126.049145:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206720:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099126.049157:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1389
00000100:00100000:1.0:1766099126.049167:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b235acde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417206848:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:3.0:1766099126.049252:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206720:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 106us (1361us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099126.049262:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1536
00010000:02020000:1.0:1766099126.049278:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099126.049354:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b235acde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417206848:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 187us (388us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099126.049366:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1389
00000100:00100000:0.0:1766099126.049513:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206784
00000100:00100000:0.0:1766099126.049557:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2261
00000100:00100000:0.0:1766099126.049562:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000083116dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417206784:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099126.049605:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000083116dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417206784:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 43us (601us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099126.049627:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2261
00010000:00010000:0.0:1766099126.143759:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099126.721895:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099126.721902:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099126.721906:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099126.721913:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3630
00000100:00080000:1.0:1766099126.721917:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099126.721923:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099126.721944:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a41834df x1851885432908480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099126.722009:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432908416:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099126.722071:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099126.722105:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099126.722123:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432908544:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099126.722162:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099126.722191:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432908480:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099126.722223:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099126.722265:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908416
00000100:00100000:0.0:1766099126.722276:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908544
00000100:00100000:2.0:1766099126.722418:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432908608:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099126.722444:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099126.722459:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099126.722463:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432908352:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099126.722506:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099126.722545:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908480
00000100:00100000:2.0:1766099126.722560:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908608
00000100:00100000:2.0:1766099126.722576:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1032
00000100:00100000:2.0:1766099126.722581:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080dc22a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432908608:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099126.722618:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080dc22a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432908608:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (174us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099126.722624:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1032
00000100:00100000:2.0:1766099126.722637:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1390
00000100:00100000:2.0:1766099126.722641:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063650c43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432908480:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099126.722729:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099126.722754:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063650c43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432908480:12345-0@lo:8:kworker/u8:2.0 Request processed in 112us (531us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099126.722759:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1390
00000100:00100000:2.0:1766099126.722779:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908352
00000100:00100000:2.0:1766099126.722797:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1537
00000100:00100000:2.0:1766099126.722800:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432908352:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099126.722823:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432908352:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (317us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099126.722828:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1537
00000100:00080000:2.0:1766099126.722871:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099126.722874:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099126.722878:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432908480:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099126.722900:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa4b7e05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432908352:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766099126.722908:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004c0ecbea x1851885432908608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099133 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099126.722916:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0ecbea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432908608:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099126.725190:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2262
00000100:00100000:0.0:1766099126.725220:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2263
00000100:00100000:0.0:1766099126.725238:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432908416:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099126.726055:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b8641416 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432908544:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099126.726131:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b8641416 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432908544:12345-0@lo:400:kworker/u8:2.0 Request processed in 78us (3968us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099126.726139:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2262
00000100:00100000:1.0:1766099126.726194:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000365d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432908544:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099126.727119:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4dd01a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432908416:12345-0@lo:400:kworker/u8:2.0 Request processed in 1888us (5045us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099126.727130:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2263
00000100:00100000:1.0:1766099126.728333:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432908416:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099127.165853:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099128.189790:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099129.213683:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099130.237714:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099131.164950:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099131.165206:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207040
00000100:00100000:1.0:1766099131.165268:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1391
00000100:00100000:1.0:1766099131.165275:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081a397a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207040:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099131.165385:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099131.165508:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081a397a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207040:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 232us (568us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099131.165516:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1391
00000100:00100000:2.0:1766099131.169356:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099131.169837:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099131.169952:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206976
00000100:00100000:3.0:1766099131.170030:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2264
00000100:00100000:3.0:1766099131.170040:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075781527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417206976:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099131.170162:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075781527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417206976:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 123us (817us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099131.170171:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2264
00000100:00100000:0.0:1766099131.170300:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417206912
00000100:00100000:0.0:1766099131.170342:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1538
00000100:00100000:0.0:1766099131.170346:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206912:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099131.170379:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417206912:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 32us (543us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099131.170386:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1538
00010000:00010000:2.0:1766099131.261805:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099131.838313:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099131.838324:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099131.838328:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099131.838339:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3635
00000100:00080000:1.0:1766099131.838344:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099131.838370:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099131.838401:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d473bfd5 x1851885432908800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099131.838507:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432908672:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099131.838602:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099131.838638:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099131.838644:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432908928:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099131.838675:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099131.838722:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ebe6253 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432908864:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099131.838753:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099131.838781:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099131.838786:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008970db1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432908736:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099131.838810:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099131.838936:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d473bfd5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432908800:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099131.838968:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099131.839008:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908864
00000100:00100000:1.0:1766099131.839058:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908736
00000100:00100000:2.0:1766099131.839070:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908928
00000100:00100000:1.0:1766099131.839087:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2265
00000100:00100000:1.0:1766099131.839095:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b09b6fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432908864:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099131.839106:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908800
00000100:00100000:2.0:1766099131.839122:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1033
00000100:00100000:2.0:1766099131.839128:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0765a5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432908928:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099131.839160:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b09b6fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432908864:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (406us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099131.839170:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2265
00000100:00100000:2.0:1766099131.839177:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0765a5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432908928:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (501us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099131.839185:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1033
00000100:00100000:2.0:1766099131.839234:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1392
00000100:00100000:2.0:1766099131.839238:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000659c82b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432908800:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099131.839304:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2266
00000100:00100000:0.0:1766099131.839309:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008077429a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432908736:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:2.0:1766099131.839336:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099131.839344:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008077429a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432908736:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (532us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099131.839351:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2266
00000100:00100000:2.0:1766099131.839375:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000659c82b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432908800:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (407us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099131.839382:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1392
00010000:00080000:0.0:1766099131.839409:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ce53f395 x1851885432908928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099138 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099131.839424:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432908928:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099131.839427:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908672
00000100:00100000:2.0:1766099131.839458:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1539
00000100:00100000:2.0:1766099131.839462:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432908672:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099131.839515:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432908672:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (915us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099131.839517:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008970db1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432908736:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099131.839527:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1539
00000100:00100000:0.0:1766099131.839537:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001ebe6253 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432908864:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099131.839570:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432908672:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099131.839591:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099131.839596:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099131.839601:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d473bfd5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432908800:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099132.286423:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099133.309720:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099134.335323:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099135.357913:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099136.286186:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099136.286240:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099136.286289:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099136.286530:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207168
00000100:00100000:1.0:1766099136.286599:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2267
00000100:00100000:1.0:1766099136.286607:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081de4e5b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417207168:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.286789:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207232
00000100:00100000:2.0:1766099136.286818:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1393
00000100:00100000:2.0:1766099136.286823:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417207232:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766099136.286927:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099136.287057:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081de4e5b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417207168:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 450us (817us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099136.287066:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2267
00000100:00100000:3.0:1766099136.289143:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207104
00000100:00100000:3.0:1766099136.289428:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1540
00000100:00100000:2.0:1766099136.289448:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417207232:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 2610us (3145us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099136.289448:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417207104:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.289500:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1393
00000100:00100000:3.0:1766099136.289583:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417207104:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 144us (3406us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099136.289594:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1540
00010000:00010000:0.0:1766099136.382356:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099136.957679:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:1.0:1766099136.957683:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099136.957689:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099136.957692:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099136.957704:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3640
00000100:00100000:2.0:1766099136.957705:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432908992:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099136.957709:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099136.957717:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099136.957741:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000923f013f x1851885432909120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099136.957793:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099136.957826:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000745e50ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432909056:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.957835:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aaac931a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432909248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099136.957891:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099136.957895:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099136.957930:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099136.957943:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002cf9e15d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432909184:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.957966:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432908992
00000100:00100000:1.0:1766099136.957974:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099136.958009:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000923f013f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432909120:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099136.958017:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1541
00000100:00100000:2.0:1766099136.958028:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432908992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099136.958042:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099136.958077:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909248
00000100:00100000:2.0:1766099136.958099:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432908992:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (309us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099136.958110:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1541
00000100:00100000:1.0:1766099136.958122:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909120
00000100:00100000:1.0:1766099136.958166:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1034
00000100:00100000:1.0:1766099136.958170:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f94d2c18 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432909248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.958188:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1394
00000100:00100000:2.0:1766099136.958194:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052c5ea72 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432909120:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099136.958216:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f94d2c18 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432909248:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (329us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099136.958223:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1034
00010000:00080000:3.0:1766099136.958253:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000aaac931a x1851885432909248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099143 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099136.958274:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aaac931a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432909248:0@lo:400:kworker/u8:2.0
00010000:02020000:2.0:1766099136.958290:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099136.958297:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432908992:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.958350:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052c5ea72 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432909120:12345-0@lo:8:kworker/u8:2.0 Request processed in 153us (309us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099136.958362:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1394
00000100:00080000:3.0:1766099136.958405:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099136.958410:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099136.958416:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000923f013f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432909120:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099136.958434:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909056
00000100:00100000:1.0:1766099136.958443:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909184
00000100:00100000:2.0:1766099136.958516:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2268
00000100:00100000:2.0:1766099136.958521:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088bd533c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432909056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099136.958543:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2269
00000100:00100000:1.0:1766099136.958549:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033470766 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432909184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099136.958562:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088bd533c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432909056:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (678us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099136.958570:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2268
00000100:00100000:1.0:1766099136.958591:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033470766 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432909184:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (618us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099136.958599:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2269
00000100:00100000:1.0:1766099136.958635:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002cf9e15d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432909184:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099136.958647:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000745e50ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432909056:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099137.406099:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099138.429825:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099139.455227:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099140.477874:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099141.406150:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099141.407439:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207296
00000100:00100000:0.0:1766099141.407529:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1542
00000100:00100000:0.0:1766099141.407536:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417207296:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099141.407609:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417207296:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 72us (1466us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099141.407616:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1542
00000100:00100000:1.0:1766099141.408267:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099141.408354:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207360
00000100:00100000:3.0:1766099141.408395:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2270
00000100:00100000:3.0:1766099141.408400:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000745e50ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417207360:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099141.408452:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000745e50ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417207360:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 51us (187us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099141.408460:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2270
00000100:00100000:3.0:1766099141.410395:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099141.411797:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207424
00000100:00100000:0.0:1766099141.411848:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1395
00000100:00100000:0.0:1766099141.411855:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207424:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766099141.411959:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099141.412142:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207424:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 286us (1750us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099141.412150:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1395
00010000:00010000:0.0:1766099141.501836:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099142.078107:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099142.078122:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099142.078125:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099142.078133:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3645
00000100:00080000:2.0:1766099142.078136:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099142.078142:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099142.078162:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bfede6c4 x1851885432909440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099142.078263:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432909376:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099142.078323:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099142.078365:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099142.078371:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a949ae5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432909504:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099142.078401:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099142.078439:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909376
00000100:00100000:1.0:1766099142.078503:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909504
00000100:00100000:3.0:1766099142.078509:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432909312:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099142.078533:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2271
00000100:00100000:1.0:1766099142.078540:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002cf9e15d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432909376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099142.078547:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099142.078564:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099142.078569:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432909568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099142.078612:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002cf9e15d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432909376:12345-0@lo:400:kworker/u8:3.0 Request processed in 70us (288us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099142.078614:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099142.078623:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2271
00000100:00100000:3.0:1766099142.078642:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432909440:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099142.078646:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2272
00000100:00100000:1.0:1766099142.078652:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063650c43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432909504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099142.078670:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099142.078688:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063650c43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432909504:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (285us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099142.078694:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2272
00000100:00100000:1.0:1766099142.078724:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909568
00000100:00100000:1.0:1766099142.078746:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909440
00000100:00100000:1.0:1766099142.078764:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1035
00000100:00100000:1.0:1766099142.078776:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8d6226a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432909568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099142.078816:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8d6226a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432909568:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (204us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099142.078823:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1035
00000100:00100000:1.0:1766099142.078844:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1396
00000100:00100000:1.0:1766099142.078848:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432909440:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099142.078970:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a949ae5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432909504:0@lo:400:kworker/u8:3.0
00010000:02020000:1.0:1766099142.078972:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099142.078982:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432909376:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099142.079011:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004aef0954 x1851885432909568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099149 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099142.079020:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432909568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099142.079197:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432909440:12345-0@lo:8:kworker/u8:3.0 Request processed in 347us (528us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099142.079204:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1396
00000100:00100000:2.0:1766099142.079367:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909312
00000100:00100000:2.0:1766099142.079403:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1543
00000100:00100000:2.0:1766099142.079407:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432909312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099142.079442:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432909312:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (897us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099142.079449:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1543
00000100:00100000:2.0:1766099142.079520:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432909312:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099142.079884:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099142.079889:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099142.079892:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432909440:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099142.525687:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099143.549748:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099144.577909:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099145.597739:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099146.524455:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099146.524560:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099146.524611:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207488
00000100:00100000:1.0:1766099146.524634:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099146.524653:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1544
00000100:00100000:0.0:1766099146.524659:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417207488:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099146.524741:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417207488:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 81us (181us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099146.524748:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1544
00000100:00100000:2.0:1766099146.524766:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207552
00000100:00100000:2.0:1766099146.524796:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2273
00000100:00100000:2.0:1766099146.524801:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000659c82b7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417207552:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099146.524831:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000659c82b7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417207552:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 29us (385us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099146.524838:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2273
00000100:00100000:3.0:1766099146.524979:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207616
00000100:00100000:3.0:1766099146.525013:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1397
00000100:00100000:3.0:1766099146.525017:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207616:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099146.525157:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099146.525195:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207616:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 177us (562us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099146.525202:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1397
00010000:00010000:2.0:1766099146.624269:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099147.198797:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099147.198804:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099147.198807:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099147.198815:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3650
00000100:00080000:2.0:1766099147.198819:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099147.198823:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099147.198843:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004aef0954 x1851885432909760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099147.198885:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432909824:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.198946:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099147.198975:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099147.198981:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432909696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.199002:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099147.199025:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432909760:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099147.199049:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099147.199091:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909824
00000100:00100000:2.0:1766099147.199127:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909696
00000100:00100000:2.0:1766099147.199144:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2274
00000100:00100000:2.0:1766099147.199149:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006c78fed0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432909824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.199188:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006c78fed0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432909824:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (244us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099147.199194:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2274
00000100:00100000:2.0:1766099147.199206:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2275
00000100:00100000:2.0:1766099147.199209:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9c2be5b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432909696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.199226:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9c2be5b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432909696:12345-0@lo:400:kworker/u8:2.0 Request processed in 16us (224us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099147.199233:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2275
00000100:00100000:2.0:1766099147.199252:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909760
00000100:00100000:2.0:1766099147.199266:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1398
00000100:00100000:2.0:1766099147.199269:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000220e4534 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432909760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099147.199365:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099147.199392:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000220e4534 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432909760:12345-0@lo:8:kworker/u8:2.0 Request processed in 122us (344us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099147.199397:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1398
00000100:00080000:2.0:1766099147.199423:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099147.199426:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099147.199429:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432909760:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099147.199454:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432909696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.199460:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432909824:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099147.199531:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a949ae5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432909888:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099147.199557:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099147.199591:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099147.199596:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432909632:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099147.199621:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099147.199646:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909632
00000100:00100000:0.0:1766099147.199670:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1545
00000100:00100000:0.0:1766099147.199673:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432909632:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099147.199699:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432909632:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (79us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099147.199705:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1545
00000100:00100000:0.0:1766099147.199734:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c12d69bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432909632:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.201155:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909888
00000100:00100000:2.0:1766099147.201998:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1036
00000100:00100000:2.0:1766099147.202005:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432909888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099147.202096:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432909888:12345-0@lo:400:kworker/u8:2.0 Request processed in 91us (2539us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099147.202102:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1036
00010000:00080000:3.0:1766099147.202175:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007a949ae5 x1851885432909888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099154 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099147.202202:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a949ae5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432909888:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099147.645864:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099148.669697:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099149.693673:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099150.719298:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099151.644850:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099151.644958:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207680
00000100:00100000:0.0:1766099151.645019:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1546
00000100:00100000:0.0:1766099151.645026:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417207680:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099151.645107:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417207680:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 80us (265us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099151.645114:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1546
00000100:00100000:0.0:1766099151.646911:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099151.647738:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207808
00000100:00100000:2.0:1766099151.647783:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1399
00000100:00100000:2.0:1766099151.647789:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207808:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766099151.647884:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099151.647948:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417207808:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 158us (1041us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099151.647955:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1399
00000100:00100000:1.0:1766099151.650457:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099151.650519:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207744
00000100:00100000:3.0:1766099151.650570:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2276
00000100:00100000:3.0:1766099151.650577:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a3c62a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417207744:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099151.650649:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a3c62a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417207744:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 71us (198us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099151.650656:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2276
00010000:00010000:0.0:1766099151.743357:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099152.317713:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099152.317720:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099152.317723:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099152.317730:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3655
00000100:00080000:2.0:1766099152.317734:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099152.317739:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099152.317758:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004aef0954 x1851885432910080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099152.317805:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008e61d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432910144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099152.317868:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099152.317901:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099152.317905:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432910016:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099152.317923:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099152.317944:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432910080:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099152.317973:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099152.318004:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910144
00000100:00100000:2.0:1766099152.318057:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910016
00000100:00100000:2.0:1766099152.318079:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2277
00000100:00100000:2.0:1766099152.318086:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c637466 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432910144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099152.318121:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069aac5df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432910208:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099152.318132:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c637466 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432910144:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (264us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099152.318140:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2277
00000100:00100000:2.0:1766099152.318156:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2278
00000100:00100000:2.0:1766099152.318160:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ac37e31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432910016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099152.318177:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099152.318187:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ac37e31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432910016:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (263us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099152.318193:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2278
00000100:00100000:0.0:1766099152.318204:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099152.318212:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432909952:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099152.318217:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910080
00000100:00100000:2.0:1766099152.318234:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910208
00000100:00100000:2.0:1766099152.318251:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1037
00000100:00100000:2.0:1766099152.318253:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432910208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099152.318276:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432910208:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (102us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099152.318281:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1037
00000100:00100000:2.0:1766099152.318289:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1400
00000100:00100000:2.0:1766099152.318292:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432910080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099152.318376:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099152.318423:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:0.0:1766099152.318460:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000069aac5df x1851885432910208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099159 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099152.318473:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069aac5df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432910208:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099152.318543:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432909952
00000100:00100000:1.0:1766099152.318575:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1547
00000100:00100000:1.0:1766099152.318579:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432909952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099152.318620:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432909952:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (198us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099152.318625:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1547
00000100:00100000:1.0:1766099152.318666:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432909952:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099152.318716:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432910016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099152.318726:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008e61d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432910144:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099152.318774:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099152.318778:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099152.318782:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432910080:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099152.319450:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432910080:12345-0@lo:8:kworker/u8:2.0 Request processed in 1154us (1474us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099152.319459:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1400
00010000:00010000:2.0:1766099152.765761:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099153.789779:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099154.813833:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099155.839272:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099156.769218:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099156.771897:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208000
00000100:00100000:0.0:1766099156.771957:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1401
00000100:00100000:0.0:1766099156.771963:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b49684c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417208000:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099156.772069:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099156.772144:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b49684c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417208000:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 180us (2932us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099156.772151:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1401
00000100:00100000:1.0:1766099156.773686:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099156.773741:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099156.774215:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207936
00000100:00100000:2.0:1766099156.774275:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2279
00000100:00100000:2.0:1766099156.774282:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000022e2f662 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417207936:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099156.774360:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000022e2f662 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417207936:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 77us (619us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099156.774369:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2279
00000100:00100000:0.0:1766099156.774700:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417207872
00000100:00100000:0.0:1766099156.774769:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1548
00000100:00100000:0.0:1766099156.774773:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417207872:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099156.774804:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417207872:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 30us (1124us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099156.774811:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1548
00010000:00010000:0.0:1766099156.863516:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099157.438615:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099157.438623:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099157.438627:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099157.438636:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3660
00000100:00080000:1.0:1766099157.438639:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099157.438645:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099157.438665:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bc4cc970 x1851885432910400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099157.438710:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432910336:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099157.438732:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432910464:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099157.438775:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099157.438790:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432910272:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099157.438829:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc4cc970 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432910400:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099157.438830:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099157.438859:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910272
00000100:00100000:3.0:1766099157.438861:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099157.438868:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099157.438870:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000e403c37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432910528:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099157.438903:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099157.438907:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1549
00000100:00100000:1.0:1766099157.438911:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910336
00000100:00100000:0.0:1766099157.438915:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432910272:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099157.438948:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2280
00000100:00100000:1.0:1766099157.438954:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085fda9a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432910336:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099157.438958:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910400
00000100:00100000:0.0:1766099157.438966:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432910272:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (139us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099157.438974:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1549
00000100:00100000:3.0:1766099157.438985:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910528
00000100:00100000:1.0:1766099157.439018:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085fda9a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432910336:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (243us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099157.439027:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2280
00000100:00100000:3.0:1766099157.439035:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1038
00000100:00100000:3.0:1766099157.439042:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432910528:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099157.439050:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432910272:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099157.439058:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1402
00000100:00100000:1.0:1766099157.439065:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000923f013f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432910400:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099157.439094:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099157.439094:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432910528:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (189us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099157.439105:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1038
00010000:00080000:0.0:1766099157.439129:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000e403c37 x1851885432910528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099164 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099157.439138:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000e403c37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432910528:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099157.439159:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099157.439163:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432910336:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099157.439181:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910464
00000100:00100000:1.0:1766099157.439198:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000923f013f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432910400:12345-0@lo:8:kworker/u8:2.0 Request processed in 133us (332us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099157.439204:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2281
00000100:00100000:0.0:1766099157.439212:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aa8ac3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432910464:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099157.439228:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1402
00000100:00100000:0.0:1766099157.439242:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aa8ac3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432910464:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (153us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099157.439248:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2281
00000100:00080000:1.0:1766099157.439268:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099157.439272:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099157.439277:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432910464:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099157.439278:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc4cc970 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432910400:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099157.889620:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099158.912381:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099159.933682:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099160.957709:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099161.884998:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099161.887093:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099161.888255:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208064
00000100:00100000:1.0:1766099161.888318:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1550
00000100:00100000:1.0:1766099161.888324:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417208064:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099161.888398:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417208064:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 73us (1311us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099161.888405:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1550
00000100:00100000:2.0:1766099161.888958:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208128
00000100:00100000:2.0:1766099161.889004:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2282
00000100:00100000:2.0:1766099161.889009:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000136070b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417208128:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099161.889055:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000136070b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417208128:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 46us (4065us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099161.889061:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2282
00000100:00100000:3.0:1766099161.891133:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099161.891948:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208192
00000100:00100000:2.0:1766099161.893001:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1403
00000100:00100000:2.0:1766099161.893012:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417208192:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099161.893118:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099161.893695:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417208192:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 680us (2568us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099161.893705:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1403
00010000:00010000:2.0:1766099161.981831:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099162.557747:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099162.557755:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099162.557759:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099162.557768:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3665
00000100:00080000:1.0:1766099162.557772:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099162.557778:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099162.557804:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000aaac931a x1851885432910720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099162.557859:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aaac931a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432910720:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099162.557940:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099162.558039:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910720
00000100:00100000:1.0:1766099162.558081:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1404
00000100:00100000:1.0:1766099162.558088:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432910720:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099162.558192:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099162.558237:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432910720:12345-0@lo:8:kworker/u8:3.0 Request processed in 149us (299us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099162.559253:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc4cc970 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432910592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099162.559318:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099162.559370:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099162.559377:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432910848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099162.559404:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099162.559445:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910592
00000100:00100000:3.0:1766099162.559514:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1551
00000100:00100000:3.0:1766099162.559520:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432910592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099162.559569:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432910592:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (252us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099162.559576:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1551
00000100:00100000:3.0:1766099162.559626:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc4cc970 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432910592:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099162.559691:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099162.559695:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099162.559698:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aaac931a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432910720:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099162.559723:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910848
00000100:00100000:3.0:1766099162.559745:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1039
00000100:00100000:3.0:1766099162.559749:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432910848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099162.559779:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432910848:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (375us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099162.559785:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1039
00010000:00080000:3.0:1766099162.559807:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ca5e8fcf x1851885432910848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099169 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099162.559817:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432910848:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099162.560000:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d473bfd5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432910784:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099162.560036:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099162.560054:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099162.560059:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432910656:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099162.560076:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099162.560104:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910784
00000100:00100000:0.0:1766099162.560139:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910656
00000100:00100000:0.0:1766099162.560169:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2283
00000100:00100000:0.0:1766099162.560172:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c44db07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432910784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099162.560210:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c44db07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432910784:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (174us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099162.560216:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2283
00000100:00100000:0.0:1766099162.560232:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2284
00000100:00100000:0.0:1766099162.560236:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aec2f11c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432910656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099162.560260:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aec2f11c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432910656:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (184us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099162.560265:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2284
00000100:00100000:0.0:1766099162.560298:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432910656:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099162.560305:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d473bfd5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432910784:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099162.563459:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1404
00010000:00010000:0.0:1766099163.007352:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099164.031170:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099165.055241:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099166.078092:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099167.012391:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099167.012436:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099167.012510:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099167.012750:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208256
00000100:00100000:2.0:1766099167.012795:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1552
00000100:00100000:2.0:1766099167.012802:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417208256:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099167.012812:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208320
00000100:00100000:0.0:1766099167.012853:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2285
00000100:00100000:0.0:1766099167.012858:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417208320:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099167.012872:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417208256:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 69us (488us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099167.012879:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1552
00000100:00100000:0.0:1766099167.012899:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417208320:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 39us (462us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099167.012907:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2285
00000100:00100000:1.0:1766099167.013302:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208384
00000100:00100000:1.0:1766099167.013339:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1405
00000100:00100000:1.0:1766099167.013343:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417208384:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099167.013446:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099167.013510:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417208384:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 165us (1000us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099167.013516:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1405
00010000:00010000:0.0:1766099167.102146:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099167.679706:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099167.679713:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099167.679716:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099167.679724:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3671
00000100:00080000:1.0:1766099167.679728:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099167.679734:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099167.679752:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d473bfd5 x1851885432911040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099167.679808:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432910976:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099167.679866:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099167.679893:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099167.679897:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432911104:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099167.679913:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099167.679977:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910976
00000100:00100000:1.0:1766099167.680017:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911104
00000100:00100000:1.0:1766099167.680044:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2286
00000100:00100000:1.0:1766099167.680049:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aaac931a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432910976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099167.680088:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aaac931a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432910976:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (223us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099167.680094:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2286
00000100:00100000:1.0:1766099167.680109:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2287
00000100:00100000:1.0:1766099167.680112:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc4cc970 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432911104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099167.680128:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc4cc970 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432911104:12345-0@lo:400:kworker/u8:2.0 Request processed in 16us (215us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099167.680133:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2287
00000100:00100000:1.0:1766099167.680164:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432911104:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099167.680172:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432910976:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099167.680539:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d473bfd5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432911040:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099167.680569:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432911168:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099167.680573:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099167.680607:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099167.680623:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008970db1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432910912:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099167.680634:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911040
00000100:00100000:3.0:1766099167.680650:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099167.680669:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911168
00000100:00100000:3.0:1766099167.680675:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432910912
00000100:00100000:0.0:1766099167.680688:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1040
00000100:00100000:0.0:1766099167.680692:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c21e5498 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432911168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099167.680694:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1553
00000100:00100000:3.0:1766099167.680697:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432910912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099167.680723:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c21e5498 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432911168:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (116us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099167.680723:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432910912:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (73us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099167.680730:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1040
00000100:00100000:3.0:1766099167.680730:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1553
00000100:00100000:0.0:1766099167.680743:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1406
00000100:00100000:0.0:1766099167.680746:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432911040:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099167.680762:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008970db1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432910912:0@lo:400:kworker/u8:2.0
00010000:02020000:0.0:1766099167.680843:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:1.0:1766099167.680855:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ca5e8fcf x1851885432911168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099174 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099167.680865:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432911168:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099167.683181:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432911040:12345-0@lo:8:kworker/u8:2.0 Request processed in 2426us (2601us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099167.683195:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1406
00000100:00080000:3.0:1766099167.684646:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099167.684656:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099167.684663:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d473bfd5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432911040:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099168.129042:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099169.150255:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099170.173807:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099171.197664:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099172.131058:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099172.131149:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099172.131154:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208448
00000100:00100000:2.0:1766099172.131201:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1554
00000100:00100000:2.0:1766099172.131209:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417208448:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099172.131262:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208512
00000100:00100000:2.0:1766099172.131283:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417208448:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 74us (232us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099172.131291:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1554
00000100:00100000:1.0:1766099172.131305:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2288
00000100:00100000:1.0:1766099172.131311:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417208512:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099172.131342:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417208512:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 31us (194us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099172.131347:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2288
00000100:00100000:0.0:1766099172.134201:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099172.134294:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208576
00000100:00100000:1.0:1766099172.134344:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1407
00000100:00100000:1.0:1766099172.134350:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1ac28d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417208576:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766099172.134443:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099172.134527:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1ac28d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417208576:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 176us (332us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099172.134535:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1407
00010000:00010000:0.0:1766099172.221764:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099172.798366:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1766099172.798369:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432911232:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099172.798374:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099172.798378:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099172.798384:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3676
00000100:00080000:1.0:1766099172.798387:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099172.798392:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099172.798406:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432911296:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099172.798414:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002a45ab0c x1851885432911360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099172.798457:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099172.798458:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432911424:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099172.798462:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099172.798511:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099172.798543:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099172.798548:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432911488:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099172.798572:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099172.798619:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911488
00000100:00100000:2.0:1766099172.798636:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432911360:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099172.798657:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1041
00000100:00100000:1.0:1766099172.798663:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e4d8abb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432911488:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099172.798674:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099172.798714:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911360
00000100:00100000:1.0:1766099172.798715:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e4d8abb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432911488:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (143us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099172.798723:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1041
00000100:00100000:1.0:1766099172.798741:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1408
00000100:00100000:1.0:1766099172.798745:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432911360:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099172.798768:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911232
00000100:00100000:2.0:1766099172.798793:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1555
00000100:00100000:2.0:1766099172.798798:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432911232:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099172.798830:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099172.798834:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432911232:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (378us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099172.798842:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1555
00000100:00100000:1.0:1766099172.798866:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432911360:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (192us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099172.798873:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1408
00000100:00080000:2.0:1766099172.798928:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099172.798934:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099172.798939:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432911360:0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766099172.798941:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000467e4c34 x1851885432911488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099179 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099172.798953:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432911488:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099172.798963:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432911232:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099172.800805:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911296
00000100:00100000:0.0:1766099172.800902:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911424
00000100:00100000:0.0:1766099172.800926:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2289
00000100:00100000:0.0:1766099172.800932:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432911296:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099172.801042:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432911296:12345-0@lo:400:kworker/u8:2.0 Request processed in 109us (2582us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099172.801049:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2289
00000100:00100000:0.0:1766099172.801067:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2290
00000100:00100000:0.0:1766099172.801085:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000e403c37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432911424:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099172.801119:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000e403c37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432911424:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (2609us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099172.801125:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2290
00000100:00100000:2.0:1766099172.801165:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432911296:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099172.802410:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432911424:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099173.246708:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099174.271733:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099175.295115:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099176.317837:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099177.246744:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099177.246799:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099177.246931:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208640
00000100:00100000:2.0:1766099177.246986:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1556
00000100:00100000:2.0:1766099177.246994:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417208640:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099177.247072:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417208640:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 78us (273us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099177.247081:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1556
00000100:00100000:1.0:1766099177.247182:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208704
00000100:00100000:1.0:1766099177.247224:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2291
00000100:00100000:1.0:1766099177.247228:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417208704:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099177.247260:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417208704:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 30us (523us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099177.247266:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2291
00000100:00100000:2.0:1766099177.248582:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099177.249048:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208768
00000100:00100000:3.0:1766099177.249233:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1409
00000100:00100000:3.0:1766099177.249240:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062b9fb1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417208768:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766099177.249331:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099177.251671:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062b9fb1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417208768:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 2424us (3087us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099177.251684:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1409
00010000:00010000:2.0:1766099177.341673:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099177.917676:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099177.917683:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099177.917686:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099177.917694:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3681
00000100:00080000:1.0:1766099177.917698:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:2.0:1766099177.917704:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432911552:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099177.917705:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099177.917728:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000467e4c34 x1851885432911680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099177.917773:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432911744:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099177.917781:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099177.917812:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099177.917818:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099177.917821:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432911808:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099177.917842:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099177.917864:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432911616:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099177.917882:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099177.917896:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099177.917928:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432911680:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099177.917942:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911552
00000100:00100000:1.0:1766099177.917950:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099177.917962:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911808
00000100:00100000:1.0:1766099177.917987:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911744
00000100:00100000:3.0:1766099177.917991:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1557
00000100:00100000:3.0:1766099177.917999:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432911552:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099177.918002:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911680
00000100:00100000:1.0:1766099177.918018:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911616
00000100:00100000:2.0:1766099177.918029:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1042
00000100:00100000:2.0:1766099177.918035:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432911808:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099177.918047:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2292
00000100:00100000:1.0:1766099177.918054:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d473bfd5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432911744:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099177.918058:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432911552:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (281us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099177.918068:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1557
00000100:00100000:2.0:1766099177.918074:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432911808:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (192us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099177.918083:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1042
00000100:00100000:1.0:1766099177.918100:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d473bfd5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432911744:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (282us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099177.918105:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1410
00000100:00100000:2.0:1766099177.918113:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432911680:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099177.918114:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2292
00010000:00080000:3.0:1766099177.918125:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002a45ab0c x1851885432911808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099184 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099177.918134:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2293
00000100:00100000:1.0:1766099177.918139:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008970db1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432911616:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099177.918141:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a45ab0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432911808:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099177.918166:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008970db1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432911616:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099177.918171:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2293
00010000:02020000:2.0:1766099177.918205:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099177.918331:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432911552:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099177.918512:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432911616:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099177.918521:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432911744:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099177.919030:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432911680:12345-0@lo:8:kworker/u8:3.0 Request processed in 918us (1080us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099177.919038:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1410
00000100:00080000:2.0:1766099177.919240:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099177.919245:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099177.919248:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432911680:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099178.369073:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099179.391640:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099180.413693:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099181.441980:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099182.369544:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099182.369574:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099182.369599:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099182.369632:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208960
00000100:00100000:2.0:1766099182.369661:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1411
00000100:00100000:2.0:1766099182.369666:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080dc22a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417208960:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766099182.369754:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099182.369802:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080dc22a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417208960:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 135us (202us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099182.369808:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1411
00000100:00100000:0.0:1766099182.370392:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208896
00000100:00100000:0.0:1766099182.370437:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2294
00000100:00100000:0.0:1766099182.370442:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b235acde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417208896:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099182.370530:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b235acde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417208896:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 87us (993us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099182.370537:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2294
00000100:00100000:1.0:1766099182.370773:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417208832
00000100:00100000:1.0:1766099182.370795:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1558
00000100:00100000:1.0:1766099182.370799:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417208832:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099182.370840:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417208832:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 41us (1266us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099182.370847:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1558
00010000:00010000:0.0:1766099182.462247:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099183.038626:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099183.038635:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099183.038638:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099183.038646:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3686
00000100:00080000:2.0:1766099183.038649:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099183.038655:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099183.038676:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bfede6c4 x1851885432912000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099183.038726:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008e61d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432911936:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.038795:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099183.038852:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099183.038857:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432912064:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.038878:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099183.038917:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912000:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099183.038941:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099183.038971:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911936
00000100:00100000:2.0:1766099183.039004:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912064
00000100:00100000:2.0:1766099183.039019:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2295
00000100:00100000:2.0:1766099183.039024:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a949ae5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432911936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.039081:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a949ae5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432911936:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (287us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099183.039087:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2295
00000100:00100000:2.0:1766099183.039100:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2296
00000100:00100000:2.0:1766099183.039103:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c12d69bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432912064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.039124:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c12d69bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432912064:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (247us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099183.039130:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2296
00000100:00100000:2.0:1766099183.039148:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912000
00000100:00100000:2.0:1766099183.039166:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1412
00000100:00100000:2.0:1766099183.039169:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432912000:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099183.039265:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099183.039295:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432912000:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (353us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099183.039301:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1412
00000100:00080000:2.0:1766099183.039334:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099183.039338:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099183.039341:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912000:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099183.039369:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432912064:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.039378:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008e61d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432911936:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099183.039459:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069aac5df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432912128:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099183.039523:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099183.039539:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099183.039544:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432911872:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099183.039571:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099183.046009:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432911872
00000100:00100000:3.0:1766099183.046077:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1559
00000100:00100000:3.0:1766099183.046084:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432911872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099183.046171:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432911872:12345-0@lo:400:kworker/u8:2.0 Request processed in 86us (6600us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099183.046180:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1559
00000100:00100000:3.0:1766099183.046235:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432911872:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.046710:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912128
00000100:00100000:2.0:1766099183.046736:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1043
00000100:00100000:2.0:1766099183.046740:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432912128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099183.046793:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432912128:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (7270us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099183.046800:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1043
00010000:00080000:1.0:1766099183.046883:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000069aac5df x1851885432912128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099190 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099183.046908:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069aac5df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432912128:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099183.485843:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099184.510994:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099185.533877:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099186.557822:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099187.494019:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099187.494067:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099187.494081:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209088
00000100:00100000:2.0:1766099187.494111:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099187.494142:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209024
00000100:00100000:0.0:1766099187.494144:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2297
00000100:00100000:0.0:1766099187.494152:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c0ecbea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417209088:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099187.494177:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1560
00000100:00100000:1.0:1766099187.494184:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417209024:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099187.494544:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209152
00000100:00100000:3.0:1766099187.494602:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1413
00000100:00100000:3.0:1766099187.494608:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069aac5df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209152:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099187.494726:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099187.496409:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c0ecbea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417209088:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 2249us (2391us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099187.496441:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2297
00000100:00100000:1.0:1766099187.496615:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417209024:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 2429us (2548us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099187.496625:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1560
00000100:00100000:3.0:1766099187.498685:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069aac5df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209152:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 4064us (4563us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099187.498703:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1413
00010000:00010000:0.0:1766099187.582392:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099188.159258:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099188.159265:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099188.159268:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099188.159275:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3691
00000100:00080000:2.0:1766099188.159279:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099188.159285:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099188.159308:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000032cf85a9 x1851885432912320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099188.159313:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099188.159344:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432912192:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099188.159378:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008e61d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432912256:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099188.159468:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099188.159471:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099188.159641:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099188.159651:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432912448:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099188.160213:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912256
00000100:00100000:1.0:1766099188.160256:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2298
00000100:00100000:1.0:1766099188.160262:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432912256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099188.161377:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099188.161389:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432912384:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099188.161431:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099188.161468:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912384
00000100:00100000:2.0:1766099188.161523:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2299
00000100:00100000:2.0:1766099188.161528:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432912384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099188.161613:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912192
00000100:00100000:0.0:1766099188.161632:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099188.161677:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1561
00000100:00100000:3.0:1766099188.161684:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432912192:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099188.161764:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432912192:12345-0@lo:400:kworker/u8:2.0 Request processed in 79us (2306us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099188.161782:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912320:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099188.161785:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1561
00000100:00100000:0.0:1766099188.161831:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099188.161878:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912448
00000100:00100000:0.0:1766099188.161919:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912320
00000100:00100000:0.0:1766099188.161947:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044
00000100:00100000:0.0:1766099188.161956:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c751a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432912448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099188.161966:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1414
00000100:00100000:3.0:1766099188.161975:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080791d40 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432912320:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099188.162000:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c751a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432912448:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (367us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099188.162007:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044
00010000:00080000:0.0:1766099188.162050:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000daafc58 x1851885432912448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099195 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099188.162062:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432912448:0@lo:400:kworker/u8:2.0
00010000:02020000:3.0:1766099188.162074:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099188.170970:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080791d40 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432912320:12345-0@lo:8:kworker/u8:2.0 Request processed in 8983us (9129us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099188.170991:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1414
00000100:00100000:0.0:1766099188.171568:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432912192:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099188.173616:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432912256:12345-0@lo:400:kworker/u8:2.0 Request processed in 13345us (14149us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099188.173634:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2298
00000100:00100000:0.0:1766099188.174430:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008e61d6e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432912256:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099188.176082:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099188.176091:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099188.176097:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032cf85a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912320:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099188.178716:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432912384:12345-0@lo:400:kworker/u8:2.0 Request processed in 17180us (17279us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099188.178730:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2299
00000100:00100000:0.0:1766099188.179579:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfede6c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432912384:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099188.605887:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099189.629949:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099190.655110:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099191.678262:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099192.608778:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099192.608835:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099192.609080:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209344
00000100:00100000:3.0:1766099192.609162:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1415
00000100:00100000:3.0:1766099192.609192:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209344:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00000100:00100000:2.0:1766099192.609299:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00010000:02020000:3.0:1766099192.609302:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099192.609384:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209280
00000100:00100000:0.0:1766099192.609454:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2300
00000100:00100000:0.0:1766099192.609460:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417209280:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099192.609614:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209216
00000100:00100000:2.0:1766099192.609643:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1562
00000100:00100000:2.0:1766099192.609648:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417209216:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099192.634835:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417209280:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 25367us (26056us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099192.634850:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2300
00000100:00100000:1.0:1766099192.635306:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417209216:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 25655us (26007us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099192.635314:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1562
00000100:00100000:3.0:1766099192.635626:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209344:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 26433us (26790us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099192.635638:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1415
00010000:00010000:2.0:1766099192.702589:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099193.278850:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099193.278858:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099193.278861:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099193.278868:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3696
00000100:00080000:1.0:1766099193.278871:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099193.278876:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099193.278876:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432912512:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099193.278895:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ad1e01cd x1851885432912640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099193.278932:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432912576:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099193.278938:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099193.278962:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099193.278963:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099193.278968:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432912768:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099193.278978:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099193.278982:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432912704:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099193.278992:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099193.279000:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099193.279026:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912576
00000100:00100000:1.0:1766099193.279033:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912704
00000100:00100000:2.0:1766099193.279066:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2301
00000100:00100000:1.0:1766099193.279071:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2302
00000100:00100000:2.0:1766099193.279071:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081a397a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432912576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099193.279075:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0765a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432912704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099193.279115:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0765a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432912704:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (113us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099193.279117:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081a397a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432912576:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (155us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099193.279122:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2302
00000100:00100000:2.0:1766099193.279124:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2301
00000100:00100000:1.0:1766099193.279155:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432912704:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099193.279158:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912640:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099193.279164:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432912576:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099193.279180:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099193.279190:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912512
00000100:00100000:2.0:1766099193.279206:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912768
00000100:00100000:1.0:1766099193.279211:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1563
00000100:00100000:1.0:1766099193.279215:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432912512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099193.279224:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912640
00000100:00100000:2.0:1766099193.279237:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1045
00000100:00100000:2.0:1766099193.279241:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432912768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099193.279259:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432912512:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (321us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099193.279266:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1563
00000100:00100000:2.0:1766099193.279266:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432912768:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (275us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099193.279271:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1045
00000100:00100000:2.0:1766099193.279281:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1416
00000100:00100000:2.0:1766099193.279284:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432912640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099193.279439:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099193.279503:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432912640:12345-0@lo:8:kworker/u8:3.0 Request processed in 217us (323us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099193.279510:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1416
00000100:00080000:1.0:1766099193.279514:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099193.279517:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099193.279521:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912640:0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766099193.279529:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002df41de4 x1851885432912768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099200 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099193.279544:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432912768:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099193.279553:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432912512:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099193.725748:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099194.757917:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099195.774244:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099196.797912:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099197.724943:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099197.724992:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099197.725044:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099197.725144:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209472
00000100:00100000:1.0:1766099197.725203:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2303
00000100:00100000:1.0:1766099197.725209:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd9b9056 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417209472:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099197.725283:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd9b9056 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417209472:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 73us (346us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099197.725291:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2303
00000100:00100000:2.0:1766099197.725397:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209408
00000100:00100000:2.0:1766099197.725432:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1564
00000100:00100000:2.0:1766099197.725436:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417209408:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766099197.725467:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417209408:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 30us (424us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099197.725516:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1564
00000100:00100000:3.0:1766099197.726231:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209536
00000100:00100000:3.0:1766099197.726269:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1417
00000100:00100000:3.0:1766099197.726274:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f760715e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209536:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766099197.726359:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099197.728406:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f760715e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209536:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 2126us (3409us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099197.728419:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1417
00010000:00010000:0.0:1766099197.821811:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099198.397821:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:1.0:1766099198.397830:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037cc3dde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432912832:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099198.397830:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099198.397834:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099198.397842:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3701
00000100:00080000:3.0:1766099198.397845:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099198.397851:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099198.397860:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432912896:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099198.397875:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ce53f395 x1851885432912960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099198.397913:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099198.397923:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099198.397946:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099198.397954:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432913088:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099198.397956:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099198.397965:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432913024:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099198.397993:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099198.398002:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099198.398016:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912832
00000100:00100000:2.0:1766099198.398041:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912896
00000100:00100000:1.0:1766099198.398065:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913088
00000100:00100000:0.0:1766099198.398068:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1565
00000100:00100000:0.0:1766099198.398075:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432912832:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099198.398081:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913024
00000100:00100000:1.0:1766099198.398104:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1046
00000100:00100000:1.0:1766099198.398110:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad1e01cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432913088:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099198.398116:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2304
00000100:00100000:2.0:1766099198.398121:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5c3581c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432912896:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099198.398141:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432912832:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (229us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099198.398149:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1565
00000100:00100000:1.0:1766099198.398172:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad1e01cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432913088:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (178us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099198.398178:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5c3581c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432912896:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (255us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099198.398185:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1046
00000100:00100000:2.0:1766099198.398189:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2304
00010000:00080000:0.0:1766099198.398204:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ca5e8fcf x1851885432913088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099205 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099198.398209:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2305
00000100:00100000:1.0:1766099198.398215:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912960:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099198.398217:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432913024:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099198.398234:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432913088:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099198.398250:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037cc3dde pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432912832:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099198.398260:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099198.398267:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432913024:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (265us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099198.398275:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2305
00000100:00100000:1.0:1766099198.398303:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432912960
00000100:00100000:1.0:1766099198.398329:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1418
00000100:00100000:1.0:1766099198.398334:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064d02b01 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432912960:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099198.398430:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099198.398470:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064d02b01 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432912960:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (212us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099198.398503:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1418
00000100:00080000:0.0:1766099198.398508:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099198.398514:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099198.398519:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432912960:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099198.398616:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000467e4c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432913024:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099198.398632:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432912896:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099198.850960:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099199.869721:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099200.894438:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099201.917807:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099202.849174:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099202.849235:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099202.849324:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099202.849411:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209728
00000100:00100000:3.0:1766099202.849454:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1419
00000100:00100000:3.0:1766099202.849460:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b63cd4b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209728:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099202.849613:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099202.849713:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b63cd4b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209728:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 252us (389us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099202.849722:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1419
00000100:00100000:1.0:1766099202.849864:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209600
00000100:00100000:1.0:1766099202.849902:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1566
00000100:00100000:1.0:1766099202.849907:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417209600:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099202.849947:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417209600:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 39us (712us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099202.849954:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1566
00000100:00100000:2.0:1766099202.850758:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209664
00000100:00100000:2.0:1766099202.850804:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2306
00000100:00100000:2.0:1766099202.850809:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec57a858 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417209664:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099202.850949:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec57a858 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417209664:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 138us (1781us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099202.850960:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2306
00010000:00010000:0.0:1766099202.941790:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099203.518010:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7e31e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432913152:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099203.518019:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099203.518025:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099203.518028:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099203.518033:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3706
00000100:00080000:0.0:1766099203.518046:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099203.518053:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099203.518075:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000064899a73 x1851885432913280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099203.518083:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099203.518116:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099203.518124:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432913408:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099203.518156:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099203.518159:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432913344:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099203.518200:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099203.518200:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0a5d210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432913216:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099203.518233:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064899a73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432913280:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099203.518256:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099203.518289:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913408
00000100:00100000:0.0:1766099203.518323:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913280
00000100:00100000:0.0:1766099203.518339:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1047
00000100:00100000:0.0:1766099203.518345:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432913408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099203.518386:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432913408:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (231us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099203.518395:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047
00000100:00100000:0.0:1766099203.518411:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1420
00000100:00100000:0.0:1766099203.518414:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000529c8629 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432913280:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099203.518536:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099203.518585:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000529c8629 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432913280:12345-0@lo:8:kworker/u8:3.0 Request processed in 169us (328us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099203.518592:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1420
00000100:00080000:0.0:1766099203.518631:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099203.518635:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099203.518639:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064899a73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432913280:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099203.518676:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913344
00000100:00100000:0.0:1766099203.518703:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2307
00000100:00100000:0.0:1766099203.518707:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003982f589 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432913344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099203.518739:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003982f589 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432913344:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (542us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099203.518746:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2307
00000100:00100000:0.0:1766099203.518773:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432913344:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099203.518786:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913152
00010000:00080000:0.0:1766099203.518803:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f68a282c x1851885432913408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099210 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099203.518813:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432913408:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099203.518818:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1567
00000100:00100000:3.0:1766099203.518825:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432913152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099203.518870:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432913152:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (791us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099203.518877:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1567
00000100:00100000:3.0:1766099203.518918:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7e31e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432913152:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099203.519044:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099203.520808:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913216
00000100:00100000:3.0:1766099203.520859:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2308
00000100:00100000:3.0:1766099203.520864:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432913216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099203.520933:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432913216:12345-0@lo:400:kworker/u8:3.0 Request processed in 69us (1890us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099203.520940:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2308
00000100:00100000:2.0:1766099203.521016:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0a5d210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432913216:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099203.967915:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099204.993878:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099206.013683:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099207.038545:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099207.965517:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099207.965567:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099207.965606:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099207.965849:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209856
00000100:00100000:2.0:1766099207.965904:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2309
00000100:00100000:2.0:1766099207.965911:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a45ab0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417209856:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099207.966438:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209920
00000100:00100000:0.0:1766099207.966468:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1421
00000100:00100000:0.0:1766099207.966473:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6ab8e56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209920:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766099207.966611:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099207.966688:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6ab8e56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417209920:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 214us (1217us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099207.966707:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1421
00000100:00100000:2.0:1766099207.966928:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a45ab0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417209856:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 1016us (1321us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099207.967345:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209792
00000100:00100000:1.0:1766099207.967387:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1568
00000100:00100000:1.0:1766099207.967392:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417209792:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099207.967426:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417209792:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (1860us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099207.967433:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1568
00000100:00100000:2.0:1766099207.968396:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2309
00010000:00010000:2.0:1766099208.061852:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099208.639630:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099208.639638:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099208.639642:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099208.639649:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3711
00000100:00080000:0.0:1766099208.639653:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099208.639659:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099208.639689:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000064899a73 x1851885432913600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099208.639755:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432913664:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099208.639819:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099208.639850:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099208.639856:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432913536:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099208.639878:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099208.639903:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064899a73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432913600:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099208.639926:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099208.639960:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913664
00000100:00100000:0.0:1766099208.639998:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913536
00000100:00100000:0.0:1766099208.640014:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2310
00000100:00100000:0.0:1766099208.640019:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5e0abc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432913664:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099208.640064:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5e0abc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432913664:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (245us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099208.640071:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2310
00000100:00100000:0.0:1766099208.640087:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2311
00000100:00100000:0.0:1766099208.640091:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583568cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432913536:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099208.640132:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583568cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432913536:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (254us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099208.640139:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2311
00000100:00100000:0.0:1766099208.640158:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913600
00000100:00100000:0.0:1766099208.640176:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1422
00000100:00100000:0.0:1766099208.640179:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c025e697 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432913600:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099208.640292:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099208.640338:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c025e697 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432913600:12345-0@lo:8:kworker/u8:3.0 Request processed in 158us (412us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099208.640345:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1422
00000100:00080000:0.0:1766099208.640383:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099208.640387:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099208.640390:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064899a73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432913600:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099208.640418:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432913536:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099208.640427:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432913664:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099208.640721:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c44ad7a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432913728:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099208.640753:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099208.640781:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099208.640786:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432913472:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099208.640811:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099208.640901:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913472
00000100:00100000:1.0:1766099208.640936:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1569
00000100:00100000:1.0:1766099208.640940:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432913472:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099208.640971:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432913472:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (160us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099208.640976:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1569
00000100:00100000:2.0:1766099208.641371:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432913472:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099208.645049:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913728
00000100:00100000:1.0:1766099208.646628:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1048
00000100:00100000:1.0:1766099208.646638:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432913728:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099208.646743:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432913728:12345-0@lo:400:kworker/u8:3.0 Request processed in 105us (5989us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099208.646750:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1048
00010000:00080000:3.0:1766099208.649076:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c44ad7a3 x1851885432913728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099215 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099208.649095:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c44ad7a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432913728:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099209.088764:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099210.109699:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099211.135211:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099212.157749:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099213.091702:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099213.092035:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099213.092076:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099213.092147:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417209984
00000100:00100000:1.0:1766099213.092163:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210048
00000100:00100000:2.0:1766099213.092214:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1570
00000100:00100000:2.0:1766099213.092223:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417209984:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099213.092264:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2312
00000100:00100000:1.0:1766099213.092273:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052c5ea72 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417210048:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099213.092668:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052c5ea72 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417210048:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 394us (632us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099213.092679:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417209984:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 446us (980us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099213.092692:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2312
00000100:00100000:2.0:1766099213.092698:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1570
00000100:00100000:1.0:1766099213.092769:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210112
00000100:00100000:1.0:1766099213.092804:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1423
00000100:00100000:1.0:1766099213.092809:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417210112:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766099213.092908:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099213.092943:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417210112:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 133us (867us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099213.092950:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1423
00010000:00010000:0.0:1766099213.181740:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099213.759637:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099213.759647:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099213.759651:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099213.759660:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3717
00000100:00080000:0.0:1766099213.759677:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099213.759704:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099213.759743:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000064899a73 x1851885432913920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099213.759808:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c44ad7a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432914048:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099213.759842:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064899a73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432913920:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099213.759842:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008f9c19d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432913984:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.760005:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099213.760012:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099213.760605:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099213.760657:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099213.760667:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432913792:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.760668:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099213.760679:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432913856:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099213.760709:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099213.760740:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099213.760764:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913920
00000100:00100000:1.0:1766099213.760765:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914048
00000100:00100000:2.0:1766099213.760810:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049
00000100:00100000:2.0:1766099213.760818:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043db2b7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432914048:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.760862:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913792
00000100:00100000:2.0:1766099213.760881:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043db2b7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432914048:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (870us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099213.760890:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049
00000100:00100000:2.0:1766099213.760916:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1424
00000100:00100000:2.0:1766099213.760921:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f68a282c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432913920:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099213.760933:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1571
00000100:00100000:3.0:1766099213.760940:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432913792:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.761004:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432913792:12345-0@lo:400:kworker/u8:3.0 Request processed in 65us (296us total) trans 0 rc 0/0
00010000:02020000:2.0:1766099213.761022:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099213.761025:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1571
00000100:00100000:2.0:1766099213.761073:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f68a282c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432913920:12345-0@lo:8:kworker/u8:3.0 Request processed in 151us (469us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099213.761080:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1424
00000100:00100000:3.0:1766099213.761093:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913984
00000100:00100000:3.0:1766099213.761157:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432913856
00000100:00100000:3.0:1766099213.761193:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2313
00000100:00100000:3.0:1766099213.761198:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000467e4c34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432913984:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.761263:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000467e4c34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432913984:12345-0@lo:400:kworker/u8:3.0 Request processed in 65us (1261us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099213.761285:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2313
00000100:00100000:3.0:1766099213.761297:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2314
00000100:00100000:3.0:1766099213.761314:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b49684c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432913856:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.761353:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b49684c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432913856:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (615us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099213.761375:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2314
00000100:00100000:3.0:1766099213.761453:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432913856:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099213.761515:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008f9c19d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432913984:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766099213.763602:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099213.763614:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099213.763622:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064899a73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432913920:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099213.763635:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432913792:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099213.763658:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c44ad7a3 x1851885432914048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099220 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099213.763670:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c44ad7a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432914048:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099214.208079:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099215.235091:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099216.253827:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099217.277935:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099218.207421:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099218.207648:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210176
00000100:00100000:2.0:1766099218.207723:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1572
00000100:00100000:2.0:1766099218.207733:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417210176:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099218.207836:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417210176:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 104us (427us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099218.207845:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1572
00000100:00100000:0.0:1766099218.208090:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099218.208164:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210240
00000100:00100000:1.0:1766099218.208210:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2315
00000100:00100000:1.0:1766099218.208215:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0a5d210 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417210240:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099218.208269:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0a5d210 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417210240:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 53us (180us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099218.208276:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2315
00000100:00100000:0.0:1766099218.217295:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099218.217426:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210304
00000100:00100000:2.0:1766099218.217508:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1425
00000100:00100000:2.0:1766099218.217516:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7e31e7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417210304:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099218.217630:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099218.223376:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7e31e7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417210304:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 5844us (6077us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099218.223398:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1425
00010000:00010000:0.0:1766099218.301958:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099218.877757:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099218.877782:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee5b88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432914112:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.877890:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099218.878094:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914112
00000100:00100000:3.0:1766099218.878162:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1573
00000100:00100000:3.0:1766099218.878169:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432914112:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.878233:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432914112:12345-0@lo:400:kworker/u8:3.0 Request processed in 63us (349us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099218.878241:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1573
00000100:00100000:3.0:1766099218.878289:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c21e5498 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432914176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.878328:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099218.878379:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914176
00000100:00100000:3.0:1766099218.878423:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2316
00000100:00100000:3.0:1766099218.878427:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000923f013f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432914176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.878525:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000923f013f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432914176:12345-0@lo:400:kworker/u8:3.0 Request processed in 94us (196us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099218.878540:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2316
00000100:00100000:3.0:1766099218.878624:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c21e5498 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432914176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.878672:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee5b88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432914112:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099218.878812:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099218.878818:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099218.878823:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099218.878828:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3722
00000100:00080000:0.0:1766099218.878830:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099218.878835:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099218.878864:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e1ac28d3 x1851885432914240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099218.878936:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080791d40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432914368:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099218.878941:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099218.878947:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090c751a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432914304:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.878969:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099218.878988:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099218.879001:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914304
00000100:00100000:0.0:1766099218.879016:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1ac28d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432914240:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099218.879023:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2317
00000100:00100000:3.0:1766099218.879026:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f760715e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432914304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099218.879040:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099218.879056:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f760715e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432914304:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (68us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099218.879062:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2317
00000100:00100000:0.0:1766099218.879071:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090c751a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432914304:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099218.879144:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914368
00000100:00100000:3.0:1766099218.879556:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914240
00000100:00100000:2.0:1766099218.879561:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1050
00000100:00100000:2.0:1766099218.879567:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432914368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099218.879578:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1426
00000100:00100000:3.0:1766099218.879581:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432914240:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099218.879610:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432914368:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (642us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099218.879617:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1050
00010000:00080000:2.0:1766099218.879643:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000080791d40 x1851885432914368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099225 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099218.879659:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080791d40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432914368:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766099218.879680:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099218.879729:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432914240:12345-0@lo:8:kworker/u8:3.0 Request processed in 146us (688us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099218.879736:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1426
00000100:00080000:2.0:1766099218.881574:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099218.881588:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099218.881596:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1ac28d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432914240:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099219.333670:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099220.351470:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099221.374089:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099222.400287:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099223.330050:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099223.330119:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099223.330157:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099223.330256:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210496
00000100:00100000:1.0:1766099223.330315:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1427
00000100:00100000:1.0:1766099223.330322:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417210496:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766099223.330431:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099223.330544:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417210496:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 221us (386us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099223.330551:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1427
00000100:00100000:3.0:1766099223.330612:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210368
00000100:00100000:3.0:1766099223.330647:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1574
00000100:00100000:3.0:1766099223.330652:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417210368:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099223.330688:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417210368:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 36us (571us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099223.330695:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1574
00000100:00100000:0.0:1766099223.330783:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210432
00000100:00100000:0.0:1766099223.330823:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2318
00000100:00100000:0.0:1766099223.330828:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000337df21d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417210432:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766099223.330861:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000337df21d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417210432:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 33us (825us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099223.330867:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2318
00010000:00010000:2.0:1766099223.421841:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099223.999125:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099223.999134:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099223.999139:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099223.999254:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3727
00000100:00080000:3.0:1766099223.999259:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099223.999609:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099223.999684:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ca5e8fcf x1851885432914560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099223.999762:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a458df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432914688:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099223.999845:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099223.999890:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099223.999896:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432914432:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099223.999928:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099223.999954:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432914560:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099223.999977:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099224.000014:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914688
00000100:00100000:3.0:1766099224.000060:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914560
00000100:00100000:3.0:1766099224.000083:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1051
00000100:00100000:3.0:1766099224.000089:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432914688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099224.000144:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432914688:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (301us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099224.000153:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1051
00000100:00100000:3.0:1766099224.000172:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1428
00000100:00100000:3.0:1766099224.000176:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432914560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099224.000281:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099224.000316:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432914560:12345-0@lo:8:kworker/u8:3.0 Request processed in 139us (338us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099224.000323:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1428
00000100:00100000:2.0:1766099224.001052:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914432
00000100:00100000:2.0:1766099224.001154:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1575
00000100:00100000:2.0:1766099224.001160:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432914432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001190:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432914432:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (1262us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099224.001197:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1575
00000100:00100000:2.0:1766099224.001230:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432914624:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001259:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099224.001282:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099224.001288:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432914496:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001308:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766099224.001361:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099224.001365:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099224.001370:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432914560:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001403:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432914432:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766099224.001416:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000086a458df x1851885432914688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099230 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099224.001428:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a458df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432914688:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001464:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914624
00000100:00100000:2.0:1766099224.001529:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914496
00000100:00100000:2.0:1766099224.001548:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2319
00000100:00100000:2.0:1766099224.001552:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432914624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001588:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432914624:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (330us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099224.001595:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2319
00000100:00100000:2.0:1766099224.001606:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2320
00000100:00100000:2.0:1766099224.001610:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000234e8a5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432914496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001631:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000234e8a5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432914496:12345-0@lo:400:kworker/u8:3.0 Request processed in 21us (323us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099224.001637:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2320
00000100:00100000:2.0:1766099224.001660:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432914496:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099224.001669:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432914624:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099224.446754:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099225.469784:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099226.496247:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099227.518749:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099228.451578:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099228.451633:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099228.451731:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210560
00000100:00100000:2.0:1766099228.451784:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1576
00000100:00100000:2.0:1766099228.451791:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417210560:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099228.451866:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417210560:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 75us (234us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099228.451874:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1576
00000100:00100000:0.0:1766099228.451961:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210624
00000100:00100000:0.0:1766099228.451996:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2321
00000100:00100000:0.0:1766099228.452000:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b602d77c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417210624:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099228.452028:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b602d77c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417210624:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 28us (458us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099228.452034:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2321
00000100:00100000:0.0:1766099228.453962:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099228.454218:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210688
00000100:00100000:1.0:1766099228.454255:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1429
00000100:00100000:1.0:1766099228.454262:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c751a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417210688:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099228.454362:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099228.454427:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c751a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417210688:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 165us (471us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099228.454435:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1429
00010000:00010000:0.0:1766099228.541928:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099229.123165:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099229.123175:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099229.123180:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099229.123190:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3732
00000100:00080000:1.0:1766099229.123194:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099229.123202:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099229.123230:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000049dffb54 x1851885432914880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099229.123301:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432914944:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099229.123385:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099229.123427:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099229.123433:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432914816:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099229.123459:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099229.123517:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432914880:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099229.123548:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099229.123591:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914944
00000100:00100000:3.0:1766099229.123684:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099229.124197:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099229.124227:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099229.124234:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432914752:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099229.124265:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099229.124296:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914880
00000100:00100000:3.0:1766099229.124327:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915008
00000100:00100000:3.0:1766099229.124353:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052
00000100:00100000:3.0:1766099229.124359:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099229.124407:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915008:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (211us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099229.124414:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052
00000100:00100000:3.0:1766099229.124432:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1430
00000100:00100000:3.0:1766099229.124435:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432914880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099229.129202:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099229.129337:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432914880:12345-0@lo:8:kworker/u8:2.0 Request processed in 4897us (5786us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099229.129352:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1430
00000100:00100000:3.0:1766099229.129428:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914752
00000100:00100000:3.0:1766099229.129499:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1577
00000100:00100000:3.0:1766099229.129506:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432914752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099229.129554:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432914752:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (5289us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099229.129560:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1577
00000100:00080000:3.0:1766099229.133289:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099229.133305:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099229.133316:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432914880:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099229.133406:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432914752:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766099229.133423:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002df41de4 x1851885432915008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099236 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099229.133436:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915008:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099229.144510:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432914816
00000100:00100000:1.0:1766099229.144566:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2322
00000100:00100000:1.0:1766099229.144574:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432914944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099229.144684:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432914944:12345-0@lo:400:kworker/u8:2.0 Request processed in 110us (21301us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099229.144691:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2322
00000100:00100000:1.0:1766099229.144711:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2323
00000100:00100000:1.0:1766099229.144715:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432914816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099229.144735:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432914816:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (21277us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099229.144740:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2323
00000100:00100000:1.0:1766099229.144792:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432914816:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099229.144804:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432914944:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099229.566370:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099230.590325:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099231.613757:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099232.637778:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099233.565016:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099233.565073:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099233.565164:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210752
00000100:00100000:3.0:1766099233.565231:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1578
00000100:00100000:3.0:1766099233.565238:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417210752:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099233.565343:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417210752:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 104us (339us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099233.565353:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1578
00000100:00100000:0.0:1766099233.565531:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210816
00000100:00100000:0.0:1766099233.565581:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2324
00000100:00100000:0.0:1766099233.565586:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417210816:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099233.565623:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000365d5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417210816:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 37us (551us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099233.565653:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2324
00000100:00100000:2.0:1766099233.565806:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099233.565960:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210880
00000100:00100000:3.0:1766099233.565993:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1431
00000100:00100000:3.0:1766099233.565997:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061acc5af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417210880:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099233.566097:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099233.566145:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061acc5af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417210880:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 146us (339us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099233.566153:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1431
00010000:00010000:0.0:1766099233.661724:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099234.245799:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099234.245809:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099234.245813:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099234.245841:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3737
00000100:00080000:3.0:1766099234.245846:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099234.245853:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099234.245880:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002eb4c88b x1851885432915200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099234.245888:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432915136:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099234.245937:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432915264:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099234.245980:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099234.246032:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be292ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915328:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099234.246062:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099234.246080:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099234.246084:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc100191 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432915072:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099234.246113:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099234.246156:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915328
00000100:00100000:0.0:1766099234.246209:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053
00000100:00100000:1.0:1766099234.246216:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099234.246218:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915328:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099234.246259:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915136
00000100:00100000:0.0:1766099234.246282:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915328:12345-0@lo:400:kworker/u8:3.0 Request processed in 64us (218us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099234.246291:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915264
00000100:00100000:0.0:1766099234.246292:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053
00000100:00100000:1.0:1766099234.246313:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2325
00000100:00100000:1.0:1766099234.246320:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c21e5498 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432915136:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099234.246327:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915072
00000100:00100000:0.0:1766099234.246358:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1579
00000100:00100000:0.0:1766099234.246363:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432915072:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099234.246367:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c21e5498 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432915136:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (388us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099234.246375:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2325
00000100:00100000:1.0:1766099234.246393:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2326
00000100:00100000:1.0:1766099234.246398:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432915264:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099234.246403:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432915072:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (286us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099234.246411:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1579
00000100:00100000:1.0:1766099234.246447:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432915264:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (231us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099234.246454:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2326
00000100:00100000:0.0:1766099234.246500:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc100191 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432915072:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099234.246520:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432915264:0@lo:400:kworker/u8:3.0
00010000:00080000:0.0:1766099234.246521:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000be292ac7 x1851885432915328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099241 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099234.246546:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be292ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915328:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099234.246575:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432915136:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099234.246615:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432915200:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099234.246641:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099234.247039:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915200
00000100:00100000:0.0:1766099234.247070:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1432
00000100:00100000:0.0:1766099234.247075:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432915200:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099234.247173:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099234.247226:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432915200:12345-0@lo:8:kworker/u8:3.0 Request processed in 150us (584us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099234.247232:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1432
00000100:00080000:3.0:1766099234.247297:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099234.247301:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099234.247306:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432915200:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099234.686752:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099235.711285:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099236.733725:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099237.759740:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099238.691199:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099238.691246:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099238.691276:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099238.691359:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211072
00000100:00100000:0.0:1766099238.691408:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1433
00000100:00100000:0.0:1766099238.691413:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417211072:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766099238.691534:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099238.691589:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417211072:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 176us (400us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099238.691596:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1433
00000100:00100000:1.0:1766099238.691971:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417210944
00000100:00100000:1.0:1766099238.694105:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1580
00000100:00100000:1.0:1766099238.694116:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417210944:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766099238.694186:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417210944:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 71us (2940us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099238.694193:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1580
00000100:00100000:3.0:1766099238.699222:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211008
00000100:00100000:3.0:1766099238.699335:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2327
00000100:00100000:3.0:1766099238.699342:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009c235b45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417211008:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766099238.699434:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009c235b45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417211008:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 92us (8158us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099238.699440:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2327
00010000:00010000:2.0:1766099238.781724:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099239.357740:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099239.357746:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099239.357749:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099239.357757:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3742
00000100:00080000:1.0:1766099239.357760:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099239.357765:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099239.357803:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000026f8c99c x1851885432915520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099239.357840:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432915584:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099239.357894:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099239.357920:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099239.357924:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cde33f60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432915456:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099239.357940:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099239.357966:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915584
00000100:00100000:1.0:1766099239.357997:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915456
00000100:00100000:1.0:1766099239.358010:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2328
00000100:00100000:1.0:1766099239.358014:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7ccce10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432915584:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099239.358052:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7ccce10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432915584:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (159us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099239.358059:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2328
00000100:00100000:1.0:1766099239.358075:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2329
00000100:00100000:1.0:1766099239.358079:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432915456:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099239.358103:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432915456:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (162us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099239.358108:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2329
00000100:00100000:1.0:1766099239.358145:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cde33f60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432915456:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099239.358155:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432915584:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099239.358231:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432915520:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099239.358260:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099239.358379:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076cc20a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915648:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099239.358436:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099239.358454:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099239.358459:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c15c4086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432915392:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099239.358500:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099239.358523:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915520
00000100:00100000:0.0:1766099239.358551:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915648
00000100:00100000:0.0:1766099239.358568:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1054
00000100:00100000:0.0:1766099239.358571:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037d37a0f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915648:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099239.358607:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037d37a0f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915648:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (171us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099239.358612:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1054
00000100:00100000:0.0:1766099239.358624:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1434
00000100:00100000:0.0:1766099239.358627:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432915520:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099239.358728:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099239.358754:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432915520:12345-0@lo:8:kworker/u8:2.0 Request processed in 126us (494us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099239.358760:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1434
00010000:00080000:0.0:1766099239.358781:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000076cc20a1 x1851885432915648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099246 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099239.358789:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076cc20a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915648:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099239.359036:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099239.359040:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099239.359043:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432915520:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099239.359234:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915392
00000100:00100000:1.0:1766099239.359264:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1581
00000100:00100000:1.0:1766099239.359269:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432915392:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099239.359304:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432915392:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (804us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099239.359309:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1581
00000100:00100000:3.0:1766099239.359371:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c15c4086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432915392:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099239.808313:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099240.829787:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099241.864708:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099242.878127:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099243.805403:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099243.805450:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099243.806319:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211200
00000100:00100000:3.0:1766099243.806383:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2330
00000100:00100000:3.0:1766099243.806403:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064899a73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417211200:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099243.806510:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064899a73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417211200:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 107us (1059us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099243.806518:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2330
00000100:00100000:3.0:1766099243.806616:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099243.807385:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211136
00000100:00100000:2.0:1766099243.807423:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1582
00000100:00100000:2.0:1766099243.807428:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417211136:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099243.807501:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417211136:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (2106us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099243.807508:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1582
00000100:00100000:1.0:1766099243.807773:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211264
00000100:00100000:1.0:1766099243.807810:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1435
00000100:00100000:1.0:1766099243.807815:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093ed14f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417211264:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099243.808968:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099243.809028:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093ed14f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417211264:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1212us (2412us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099243.809038:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1435
00010000:00010000:0.0:1766099243.902680:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099244.478446:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432915776:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099244.478533:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099244.479226:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432915712:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099244.479267:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099244.479324:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915712
00000100:00100000:0.0:1766099244.479372:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1583
00000100:00100000:0.0:1766099244.479378:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432915712:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099244.481262:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099244.481269:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099244.481273:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099244.481278:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3747
00000100:00080000:1.0:1766099244.481281:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099244.481286:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099244.481313:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003837ea46 x1851885432915840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099244.483885:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099244.483903:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432915904:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099244.483970:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099244.484029:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915776
00000100:00100000:2.0:1766099244.484068:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915904
00000100:00100000:2.0:1766099244.484091:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2331
00000100:00100000:2.0:1766099244.484097:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432915776:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099244.484124:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2332
00000100:00100000:1.0:1766099244.484131:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb7aab30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432915904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099244.484143:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432915776:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (5609us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099244.484152:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2331
00000100:00100000:1.0:1766099244.484174:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb7aab30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432915904:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (204us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099244.484182:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2332
00000100:00100000:2.0:1766099244.484207:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432915904:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099244.484208:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008358d193 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915968:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099244.484223:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432915776:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099244.484240:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099244.484277:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915968
00000100:00100000:1.0:1766099244.484300:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1055
00000100:00100000:1.0:1766099244.484303:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099244.484330:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432915968:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (94us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099244.484334:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1055
00010000:00080000:1.0:1766099244.484371:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008358d193 x1851885432915968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099251 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099244.484380:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008358d193 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432915968:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099244.484388:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432915840:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099244.484418:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099244.486583:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432915712:12345-0@lo:400:kworker/u8:2.0 Request processed in 7201us (7313us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099244.486593:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1583
00000100:00100000:1.0:1766099244.486912:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432915840
00000100:00100000:1.0:1766099244.486956:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1436
00000100:00100000:1.0:1766099244.486962:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432915840:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099244.487086:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099244.487124:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432915840:12345-0@lo:8:kworker/u8:2.0 Request processed in 162us (2707us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099244.487131:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1436
00000100:00080000:1.0:1766099244.487171:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099244.487174:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099244.487179:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432915840:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099244.487361:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432915712:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099244.925769:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099245.950374:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099246.973678:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099247.998578:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099248.938167:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099248.938216:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099248.938826:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211456
00000100:00100000:2.0:1766099248.939209:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099248.939303:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211392
00000100:00100000:3.0:1766099248.939354:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211328
00000100:00100000:3.0:1766099248.939390:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1584
00000100:00100000:3.0:1766099248.939396:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417211328:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099248.939461:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417211328:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 64us (253us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099248.939467:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1584
00000100:00100000:1.0:1766099248.939915:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1437
00000100:00100000:1.0:1766099248.939921:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417211456:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099248.940017:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099248.940145:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2333
00000100:00100000:2.0:1766099248.940150:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a60be776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417211392:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099248.940199:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417211456:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 277us (2040us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099248.940207:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1437
00000100:00100000:2.0:1766099248.940842:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a60be776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417211392:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 690us (2625us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099248.940852:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2333
00010000:00010000:0.0:1766099249.023996:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099249.601071:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099249.601077:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099249.601081:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099249.601088:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3752
00000100:00080000:2.0:1766099249.601091:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099249.601096:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099249.601116:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001c450cdd x1851885432916160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099249.601127:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099249.601141:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432916096:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099249.601175:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c450cdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432916160:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099249.601209:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099249.601240:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000838122d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432916224:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099249.601405:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432916032:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099249.601519:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000116a4423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432916288:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099249.602994:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099249.603004:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099249.603071:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916096
00000100:00100000:0.0:1766099249.603078:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916224
00000100:00100000:1.0:1766099249.603115:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2334
00000100:00100000:1.0:1766099249.603123:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003837ea46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432916096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099249.603125:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2335
00000100:00100000:0.0:1766099249.603132:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432916224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099249.603183:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432916224:12345-0@lo:400:kworker/u8:3.0 Request processed in 50us (193us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099249.603187:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003837ea46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432916096:12345-0@lo:400:kworker/u8:3.0 Request processed in 63us (1978us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099249.603194:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2335
00000100:00100000:1.0:1766099249.603556:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2334
00000100:00100000:1.0:1766099249.603603:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000838122d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432916224:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099249.603613:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432916096:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099249.603656:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099249.603690:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916288
00000100:00100000:3.0:1766099249.603705:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916032
00000100:00100000:1.0:1766099249.603721:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056
00000100:00100000:1.0:1766099249.603726:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432916288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099249.603741:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1585
00000100:00100000:3.0:1766099249.603746:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432916032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099249.603773:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432916288:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (117us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099249.603780:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056
00000100:00100000:3.0:1766099249.603794:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432916032:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (790us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099249.603802:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1585
00010000:00080000:1.0:1766099249.603815:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000116a4423 x1851885432916288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099256 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099249.603832:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000116a4423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432916288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099249.603841:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432916032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099249.603907:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099249.605912:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916160
00000100:00100000:1.0:1766099249.605963:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1438
00000100:00100000:1.0:1766099249.605969:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432916160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099249.606079:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099249.606144:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432916160:12345-0@lo:8:kworker/u8:3.0 Request processed in 174us (2237us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099249.606151:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1438
00000100:00080000:2.0:1766099249.608829:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099249.608839:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099249.608846:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c450cdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432916160:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099250.049229:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099251.069720:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099252.094372:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099253.120021:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099254.046703:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099254.046758:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099254.046948:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211584
00000100:00100000:1.0:1766099254.047031:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2336
00000100:00100000:1.0:1766099254.047040:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c450cdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417211584:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099254.047125:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c450cdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417211584:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 86us (436us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099254.047133:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2336
00000100:00100000:3.0:1766099254.047530:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211520
00000100:00100000:3.0:1766099254.047594:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1586
00000100:00100000:3.0:1766099254.047600:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417211520:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099254.047686:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417211520:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 85us (929us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099254.047695:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1586
00000100:00100000:2.0:1766099254.047908:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099254.047978:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211648
00000100:00100000:3.0:1766099254.048005:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1439
00000100:00100000:3.0:1766099254.048010:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417211648:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099254.048132:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099254.048181:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417211648:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 170us (275us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099254.048188:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1439
00010000:00010000:2.0:1766099254.141899:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099254.721605:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099254.721616:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099254.721621:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099254.721632:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3758
00000100:00080000:3.0:1766099254.721637:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 6s
00000100:00080000:3.0:1766099254.721644:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099254.721675:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000062653d94 x1851885432916480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099254.721710:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432916416:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099254.721739:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432916544:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099254.721800:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099254.721895:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916416
00000100:00100000:0.0:1766099254.721953:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2337
00000100:00100000:0.0:1766099254.721960:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f9c19d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432916416:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099254.722021:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f9c19d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432916416:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (223us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099254.722029:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2337
00000100:00100000:0.0:1766099254.722093:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432916416:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099254.722188:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099254.722317:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432916608:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099254.722370:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099254.722378:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916544
00000100:00100000:2.0:1766099254.722415:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099254.722421:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2338
00000100:00100000:2.0:1766099254.722428:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057a445fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432916352:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099254.722429:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432916544:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099254.723865:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062653d94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432916480:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099254.724362:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099254.724394:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432916544:12345-0@lo:400:kworker/u8:2.0 Request processed in 1954us (2198us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099254.724411:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2338
00000100:00100000:2.0:1766099254.724518:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099254.724873:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916608
00000100:00100000:3.0:1766099254.724928:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916480
00000100:00100000:3.0:1766099254.724956:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1057
00000100:00100000:3.0:1766099254.724966:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000f47354 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432916608:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099254.725001:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916352
00000100:00100000:0.0:1766099254.725076:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1587
00000100:00100000:3.0:1766099254.725084:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000f47354 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432916608:12345-0@lo:400:kworker/u8:2.0 Request processed in 112us (2710us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099254.725095:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432916352:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099254.725101:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1057
00000100:00100000:3.0:1766099254.725134:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1440
00000100:00100000:3.0:1766099254.725140:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c68b1dd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432916480:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099254.725193:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432916352:12345-0@lo:400:kworker/u8:2.0 Request processed in 97us (670us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099254.725205:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1587
00010000:02020000:3.0:1766099254.725256:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099254.725331:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c68b1dd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432916480:12345-0@lo:8:kworker/u8:2.0 Request processed in 190us (972us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099254.725341:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1440
00000100:00100000:3.0:1766099254.725421:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432916544:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099254.725499:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099254.725503:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099254.725507:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062653d94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432916480:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099254.725708:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057a445fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432916352:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766099254.725727:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006e177295 x1851885432916608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099261 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099254.725738:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432916608:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099255.165822:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099256.195615:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099257.213798:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099258.237795:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099259.164755:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099259.164905:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211840
00000100:00100000:1.0:1766099259.164967:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1441
00000100:00100000:1.0:1766099259.164975:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417211840:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099259.165079:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099259.165168:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417211840:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 193us (426us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099259.165177:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1441
00000100:00100000:2.0:1766099259.165930:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099259.165975:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099259.166046:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211776
00000100:00100000:0.0:1766099259.166096:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2339
00000100:00100000:0.0:1766099259.166102:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066062a18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417211776:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099259.166170:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066062a18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417211776:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (244us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099259.166178:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2339
00000100:00100000:3.0:1766099259.166438:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211712
00000100:00100000:3.0:1766099259.166515:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1588
00000100:00100000:3.0:1766099259.166520:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417211712:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099259.166565:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417211712:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 44us (590us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099259.166594:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1588
00010000:00010000:0.0:1766099259.262214:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099259.837849:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432916672:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099259.837851:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099259.837861:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099259.837866:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099259.837872:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3763
00000100:00080000:0.0:1766099259.837877:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 11s
00000100:00080000:0.0:1766099259.837884:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099259.837911:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000080791d40 x1851885432916800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099259.837958:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099259.837984:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432916928:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099259.838022:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1ac28d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432916736:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099259.838093:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099259.838095:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099259.838284:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099259.838292:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432916864:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099259.838324:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099259.838372:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916736
00000100:00100000:0.0:1766099259.838387:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916672
00000100:00100000:1.0:1766099259.838424:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916864
00000100:00100000:0.0:1766099259.838437:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1589
00000100:00100000:0.0:1766099259.838445:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432916672:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099259.838459:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2340
00000100:00100000:1.0:1766099259.838465:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047f8d5e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432916736:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099259.838560:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047f8d5e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432916736:12345-0@lo:400:kworker/u8:2.0 Request processed in 94us (476us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099259.838567:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432916672:12345-0@lo:400:kworker/u8:2.0 Request processed in 121us (614us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099259.838572:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2340
00000100:00100000:0.0:1766099259.838581:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1589
00000100:00100000:1.0:1766099259.838604:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2341
00000100:00100000:1.0:1766099259.838612:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a8e1966 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432916864:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099259.838639:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080791d40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432916800:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099259.838656:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a8e1966 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432916864:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (330us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099259.838659:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916928
00000100:00100000:1.0:1766099259.838666:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2341
00000100:00100000:0.0:1766099259.838675:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099259.838726:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916800
00000100:00100000:0.0:1766099259.838744:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1058
00000100:00100000:1.0:1766099259.838748:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432916864:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099259.838753:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432916928:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099259.838768:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1442
00000100:00100000:1.0:1766099259.838773:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1ac28d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432916736:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099259.838780:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432916800:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099259.838803:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432916928:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (720us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099259.838812:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1058
00000100:00100000:1.0:1766099259.838830:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce4ad370 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432916672:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766099259.842626:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a8d6226a x1851885432916928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099266 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099259.842646:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432916928:0@lo:400:kworker/u8:2.0
00010000:02020000:3.0:1766099259.844356:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099259.845195:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432916800:12345-0@lo:8:kworker/u8:2.0 Request processed in 6414us (6517us total) trans 0 rc -19/-19
00000100:00080000:0.0:1766099259.845199:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099259.845203:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099259.845208:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080791d40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432916800:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099259.845208:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1442
00010000:00010000:2.0:1766099260.286667:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099261.311040:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099262.333758:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099263.358585:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099264.286284:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099264.286323:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099264.286360:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099264.286446:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211968
00000100:00100000:1.0:1766099264.286520:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2342
00000100:00100000:1.0:1766099264.286528:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d99686c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417211968:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099264.286591:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d99686c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417211968:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 62us (268us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099264.286598:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2342
00000100:00100000:3.0:1766099264.286702:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212032
00000100:00100000:3.0:1766099264.286731:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1443
00000100:00100000:3.0:1766099264.286735:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094de82c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417212032:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099264.286824:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099264.286853:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094de82c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417212032:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 117us (494us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099264.286859:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1443
00000100:00100000:0.0:1766099264.291066:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417211904
00000100:00100000:0.0:1766099264.291137:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1590
00000100:00100000:0.0:1766099264.291143:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417211904:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099264.291412:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417211904:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 268us (5136us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099264.291420:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1590
00010000:00010000:0.0:1766099264.381765:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099264.798819:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099264.798933:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212096
00000100:00100000:1.0:1766099264.798990:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1289
00000100:00100000:1.0:1766099264.798997:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417212096:12345-192.168.203.52@tcp:17:kworker/1:2.0
00000100:00100000:1.0:1766099264.799090:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417212096:12345-192.168.203.52@tcp:17:kworker/1:2.0 Request processed in 93us (279us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099264.799098:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1289
00000100:00080000:0.0:1766099264.960466:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099264.960492:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099264.960498:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099264.960508:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3768
00000100:00080000:0.0:1766099264.960512:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 16s
00000100:00080000:0.0:1766099264.960518:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099264.960539:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d779487b x1851885432917120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099264.960587:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432917184:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099264.960658:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099264.960691:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099264.960696:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432917056:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099264.960716:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099264.960736:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d779487b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432917120:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099264.960759:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099264.960790:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917184
00000100:00100000:0.0:1766099264.960826:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917056
00000100:00100000:0.0:1766099264.960845:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2343
00000100:00100000:0.0:1766099264.960850:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432917184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099264.960895:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432917184:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (237us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099264.960904:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2343
00000100:00100000:0.0:1766099264.960920:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2344
00000100:00100000:0.0:1766099264.960923:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432917056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099264.960946:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432917056:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (229us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099264.960952:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2344
00000100:00100000:0.0:1766099264.960986:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432917056:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099264.960995:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432917184:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961154:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917120
00000100:00100000:1.0:1766099264.961187:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1444
00000100:00100000:1.0:1766099264.961190:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432917120:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099264.961287:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099264.961322:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432917120:12345-0@lo:8:kworker/u8:2.0 Request processed in 131us (563us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099264.961329:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1444
00000100:00080000:1.0:1766099264.961355:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099264.961358:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099264.961362:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d779487b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432917120:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961376:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432916992:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961403:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099264.961421:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099264.961426:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a6f4b92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432917248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961445:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099264.961465:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432916992
00000100:00100000:1.0:1766099264.961521:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1591
00000100:00100000:1.0:1766099264.961525:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432916992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961554:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432916992:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (151us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099264.961560:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1591
00000100:00100000:1.0:1766099264.961590:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917248
00000100:00100000:1.0:1766099264.961608:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1059
00000100:00100000:1.0:1766099264.961611:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432917248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961634:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432917248:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (189us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099264.961639:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1059
00010000:00080000:1.0:1766099264.961664:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005a6f4b92 x1851885432917248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099271 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099264.961674:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a6f4b92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432917248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099264.961682:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432916992:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099265.407488:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099266.429717:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099267.453962:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099268.481397:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099269.408576:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099269.408630:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099269.408760:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212288
00000100:00100000:0.0:1766099269.408814:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1445
00000100:00100000:0.0:1766099269.408821:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417212288:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099269.408919:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099269.408978:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417212288:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 157us (409us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099269.408986:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1445
00000100:00100000:1.0:1766099269.410788:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099269.410982:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212224
00000100:00100000:2.0:1766099269.411092:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212160
00000100:00100000:2.0:1766099269.411122:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1592
00000100:00100000:2.0:1766099269.411127:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417212160:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099269.411207:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417212160:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 79us (2577us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099269.411214:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1592
00000100:00100000:0.0:1766099269.411661:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2345
00000100:00100000:0.0:1766099269.411667:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041477718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417212224:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099269.411717:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041477718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417212224:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 50us (934us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099269.411727:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2345
00010000:00010000:2.0:1766099269.503270:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099270.078081:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099270.078088:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099270.078092:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099270.078098:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3773
00000100:00100000:1.0:1766099270.078100:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003520602a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432917376:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099270.078103:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099270.078111:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099270.078132:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000043db2b7b x1851885432917440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099270.078188:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099270.078194:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061acc5af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432917568:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099270.078222:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099270.078229:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099270.078231:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9759d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432917504:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099270.078250:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099270.078263:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003dc105a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432917312:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099270.078279:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099270.078295:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099270.078313:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043db2b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432917440:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099270.078336:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917312
00000100:00100000:1.0:1766099270.078341:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099270.078378:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1593
00000100:00100000:1.0:1766099270.078380:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917376
00000100:00100000:2.0:1766099270.078385:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432917312:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099270.078414:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917504
00000100:00100000:2.0:1766099270.078427:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432917312:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (133us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099270.078434:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1593
00000100:00100000:1.0:1766099270.078452:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2346
00000100:00100000:1.0:1766099270.078457:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000744439a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432917376:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099270.078499:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003dc105a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432917312:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099270.078521:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000744439a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432917376:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (334us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099270.078528:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2346
00000100:00100000:2.0:1766099270.078542:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917568
00000100:00100000:1.0:1766099270.078545:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2347
00000100:00100000:1.0:1766099270.078550:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbb82e8f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432917504:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099270.078567:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917440
00000100:00100000:1.0:1766099270.078583:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbb82e8f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432917504:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (304us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099270.078586:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1060
00000100:00100000:2.0:1766099270.078592:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432917568:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099270.078595:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2347
00000100:00100000:2.0:1766099270.078630:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432917568:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (401us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099270.078634:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1446
00000100:00100000:2.0:1766099270.078641:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1060
00000100:00100000:1.0:1766099270.078642:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432917440:12345-0@lo:8:kworker/u8:2.0
00010000:00080000:2.0:1766099270.078682:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000061acc5af x1851885432917568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099277 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099270.078691:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061acc5af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432917568:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099270.078735:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099270.078779:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432917440:12345-0@lo:8:kworker/u8:2.0 Request processed in 139us (439us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099270.078786:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1446
00000100:00100000:1.0:1766099270.078813:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9759d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432917504:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099270.078825:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003520602a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432917376:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099270.081782:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099270.081792:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099270.081800:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043db2b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432917440:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099270.526530:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099271.550448:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099272.575489:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099273.602090:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099274.530286:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099274.530347:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099274.530383:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099274.530544:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212416
00000100:00100000:0.0:1766099274.530635:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2348
00000100:00100000:0.0:1766099274.530644:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005834876b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417212416:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099274.530701:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212480
00000100:00100000:2.0:1766099274.530741:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1447
00000100:00100000:2.0:1766099274.530748:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a81cfa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417212480:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:0.0:1766099274.530764:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005834876b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417212416:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 109us (370us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099274.530777:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2348
00010000:02020000:2.0:1766099274.530858:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099274.530910:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a81cfa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417212480:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 161us (630us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099274.530919:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1447
00000100:00100000:3.0:1766099274.531518:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212352
00000100:00100000:3.0:1766099274.531571:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1594
00000100:00100000:3.0:1766099274.531576:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417212352:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099274.531614:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417212352:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 38us (1268us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099274.531640:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1594
00010000:00010000:0.0:1766099274.633207:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099275.200356:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099275.200366:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099275.200370:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099275.200381:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3778
00000100:00080000:0.0:1766099275.200386:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099275.200394:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099275.200422:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a8d6226a x1851885432917760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099275.200564:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432917824:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.200662:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099275.200707:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099275.200713:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432917696:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.200736:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099275.200776:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432917760:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099275.200808:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099275.200851:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917824
00000100:00100000:2.0:1766099275.200901:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917696
00000100:00100000:2.0:1766099275.200921:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2349
00000100:00100000:2.0:1766099275.200927:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043db2b7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432917824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.200982:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043db2b7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432917824:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (320us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099275.200990:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2349
00000100:00100000:2.0:1766099275.201008:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2350
00000100:00100000:2.0:1766099275.201012:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061acc5af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432917696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201037:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061acc5af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432917696:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (301us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099275.201043:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2350
00000100:00100000:2.0:1766099275.201061:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917760
00000100:00100000:2.0:1766099275.201077:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1448
00000100:00100000:2.0:1766099275.201080:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003dc105a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432917760:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099275.201183:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099275.201217:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003dc105a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432917760:12345-0@lo:8:kworker/u8:3.0 Request processed in 136us (410us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099275.201228:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1448
00000100:00080000:2.0:1766099275.201261:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099275.201264:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099275.201268:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432917760:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201293:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432917696:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201300:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432917824:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201316:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432917632:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201345:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099275.201369:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917632
00000100:00100000:2.0:1766099275.201391:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1595
00000100:00100000:2.0:1766099275.201394:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432917632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201421:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432917632:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (76us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099275.201427:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1595
00000100:00100000:2.0:1766099275.201460:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432917632:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201468:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099275.201473:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432917888:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201520:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099275.201539:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917888
00000100:00100000:2.0:1766099275.201560:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1061
00000100:00100000:2.0:1766099275.201563:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432917888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099275.201589:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432917888:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (68us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099275.201598:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1061
00010000:00080000:2.0:1766099275.201622:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bf96017c x1851885432917888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099282 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099275.201632:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432917888:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099275.646193:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099276.669790:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099277.693681:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099278.718007:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099279.651316:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099279.651357:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099279.651438:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099279.651505:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212672
00000100:00100000:2.0:1766099279.651559:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1449
00000100:00100000:2.0:1766099279.651565:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417212672:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099279.651669:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099279.651692:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212608
00000100:00100000:1.0:1766099279.651726:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2351
00000100:00100000:2.0:1766099279.651726:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417212672:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 160us (288us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099279.651730:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417212608:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099279.651734:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1449
00000100:00100000:1.0:1766099279.651766:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417212608:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 34us (408us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099279.651772:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2351
00000100:00100000:3.0:1766099279.652263:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212544
00000100:00100000:3.0:1766099279.652305:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1596
00000100:00100000:3.0:1766099279.652310:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417212544:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099279.652356:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417212544:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 45us (1048us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099279.652363:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1596
00010000:00010000:0.0:1766099279.741758:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099280.318530:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099280.318539:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099280.318557:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099280.318569:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3783
00000100:00080000:1.0:1766099280.318573:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099280.318592:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099280.318622:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005f11e586 x1851885432918080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099280.318676:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918016:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.318765:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099280.318812:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099280.318818:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432918144:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.318842:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099280.318872:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005f11e586 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432918080:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099280.318902:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099280.318942:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918016
00000100:00100000:1.0:1766099280.318991:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918144
00000100:00100000:1.0:1766099280.319014:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2352
00000100:00100000:1.0:1766099280.319020:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.319131:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918016:12345-0@lo:400:kworker/u8:3.0 Request processed in 110us (367us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099280.319156:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2352
00000100:00100000:1.0:1766099280.319189:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2353
00000100:00100000:1.0:1766099280.319193:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c68b1dd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432918144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.319220:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c68b1dd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432918144:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (377us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099280.319225:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2353
00000100:00100000:1.0:1766099280.319294:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432918144:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.319306:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918016:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099280.319446:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5132164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432917952:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099280.319519:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099280.319541:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099280.319546:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a7d46da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432918208:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099280.319571:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099280.319600:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918080
00000100:00100000:3.0:1766099280.319623:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918208
00000100:00100000:3.0:1766099280.319657:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1062
00000100:00100000:3.0:1766099280.319661:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432918208:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099280.319695:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432918208:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (124us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099280.319702:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1062
00000100:00100000:3.0:1766099280.319715:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1450
00000100:00100000:3.0:1766099280.319718:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432918080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099280.319891:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099280.319939:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432918080:12345-0@lo:8:kworker/u8:3.0 Request processed in 219us (1037us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099280.319996:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1450
00000100:00080000:3.0:1766099280.320054:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099280.320058:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099280.320062:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005f11e586 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432918080:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766099280.320106:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001a7d46da x1851885432918208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099287 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099280.320120:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a7d46da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432918208:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.322103:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432917952
00000100:00100000:1.0:1766099280.324416:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1597
00000100:00100000:1.0:1766099280.324432:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432917952:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099280.328267:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432917952:12345-0@lo:400:kworker/u8:3.0 Request processed in 3824us (8734us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099280.328290:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1597
00000100:00100000:0.0:1766099280.329104:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5132164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432917952:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099280.765770:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099281.793354:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099282.817569:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099283.837991:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099284.787049:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099284.787198:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212864
00000100:00100000:2.0:1766099284.787257:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1451
00000100:00100000:2.0:1766099284.787264:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417212864:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099284.787361:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099284.787440:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417212864:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 176us (404us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099284.787447:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1451
00000100:00100000:0.0:1766099284.796318:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099284.796379:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099284.806778:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212736
00000100:00100000:1.0:1766099284.806848:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1598
00000100:00100000:1.0:1766099284.806856:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417212736:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099284.806950:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417212736:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (10644us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099284.806958:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1598
00000100:00100000:2.0:1766099284.812637:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212800
00000100:00100000:2.0:1766099284.812709:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2354
00000100:00100000:2.0:1766099284.812717:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417212800:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099284.812816:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417212800:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 99us (16436us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099284.812823:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2354
00010000:00010000:2.0:1766099284.861675:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099285.437743:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099285.437751:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099285.437754:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099285.437763:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3788
00000100:00080000:1.0:1766099285.437767:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099285.437773:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099285.437799:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ce45f83d x1851885432918400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099285.437871:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432918400:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099285.437877:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432918464:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099285.437976:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918336:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099285.438036:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432918528:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099285.438918:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099285.438983:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918464
00000100:00100000:1.0:1766099285.439025:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2355
00000100:00100000:1.0:1766099285.439030:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076cc20a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432918464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099285.439087:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076cc20a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432918464:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (172us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099285.439094:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2355
00000100:00100000:1.0:1766099285.439134:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432918464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099285.439391:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099285.439413:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432918272:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099285.439461:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099285.439467:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918400
00000100:00100000:1.0:1766099285.439538:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918272
00000100:00100000:3.0:1766099285.439545:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1452
00000100:00100000:3.0:1766099285.439554:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432918400:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099285.439577:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1599
00000100:00100000:1.0:1766099285.439583:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432918272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099285.439642:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432918272:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (184us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099285.439649:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1599
00010000:02020000:3.0:1766099285.439659:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099285.439689:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432918272:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099285.439726:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432918400:12345-0@lo:8:kworker/u8:3.0 Request processed in 170us (340us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099285.439736:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1452
00000100:00080000:3.0:1766099285.439904:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099285.439909:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099285.439916:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432918400:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099285.440594:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099285.444894:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099285.445096:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918528
00000100:00100000:3.0:1766099285.445155:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1063
00000100:00100000:3.0:1766099285.445162:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432918528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099285.445310:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432918528:12345-0@lo:400:kworker/u8:3.0 Request processed in 147us (423us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099285.445332:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1063
00010000:00080000:2.0:1766099285.445424:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000026f8c99c x1851885432918528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099292 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099285.445438:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432918528:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099285.447980:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918336
00000100:00100000:1.0:1766099285.448046:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2356
00000100:00100000:1.0:1766099285.448060:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099285.448153:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918336:12345-0@lo:400:kworker/u8:3.0 Request processed in 92us (7560us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099285.448160:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2356
00000100:00100000:0.0:1766099285.448387:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918336:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099285.885712:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099286.909680:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099287.933705:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099288.957787:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099289.890367:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099289.890424:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099289.890468:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099289.890637:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212928
00000100:00100000:1.0:1766099289.890692:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1600
00000100:00100000:1.0:1766099289.890700:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417212928:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099289.890766:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417212928:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 66us (342us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099289.890773:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1600
00000100:00100000:2.0:1766099289.890873:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213056
00000100:00100000:2.0:1766099289.890909:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1453
00000100:00100000:2.0:1766099289.890914:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8d6226a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417213056:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099289.891011:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099289.891059:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8d6226a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417213056:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 144us (590us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099289.891066:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1453
00000100:00100000:3.0:1766099289.891634:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417212992
00000100:00100000:3.0:1766099289.891670:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2357
00000100:00100000:3.0:1766099289.891675:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417212992:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099289.891722:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417212992:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (1362us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099289.891729:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2357
00010000:00010000:0.0:1766099289.981755:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099290.557853:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1766099290.557858:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000220e4534 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432918592:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099290.557864:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099290.557869:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099290.557880:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3793
00000100:00080000:0.0:1766099290.557885:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099290.557893:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099290.557921:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c2880771 x1851885432918720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099290.557954:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099290.557992:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099290.558000:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432918848:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099290.558020:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918656:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558040:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099290.558093:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099290.558104:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432918720:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099290.558130:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099290.558141:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062b9fb1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432918784:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558148:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099290.558183:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099290.558217:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918848
00000100:00100000:0.0:1766099290.558236:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918656
00000100:00100000:3.0:1766099290.558282:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918720
00000100:00100000:0.0:1766099290.558288:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918784
00000100:00100000:3.0:1766099290.558320:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064
00000100:00100000:0.0:1766099290.558328:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2358
00000100:00100000:3.0:1766099290.558336:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432918848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099290.558337:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080dc22a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099290.558407:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080dc22a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918656:12345-0@lo:400:kworker/u8:3.0 Request processed in 71us (316us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099290.558414:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2358
00000100:00100000:3.0:1766099290.558426:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432918848:12345-0@lo:400:kworker/u8:3.0 Request processed in 90us (383us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099290.558436:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2359
00000100:00100000:3.0:1766099290.558442:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064
00000100:00100000:0.0:1766099290.558450:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069aac5df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432918784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558472:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1454
00000100:00100000:3.0:1766099290.558535:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432918720:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099290.558557:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069aac5df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432918784:12345-0@lo:400:kworker/u8:3.0 Request processed in 108us (375us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099290.558565:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2359
00000100:00100000:0.0:1766099290.558590:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918592
00000100:00100000:0.0:1766099290.558612:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1601
00000100:00100000:0.0:1766099290.558616:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432918592:12345-0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766099290.558645:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099290.558646:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432918592:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (695us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099290.558653:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1601
00000100:00100000:0.0:1766099290.558698:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062b9fb1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432918784:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558735:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432918720:12345-0@lo:8:kworker/u8:3.0 Request processed in 199us (587us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099290.558737:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918656:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558757:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1454
00000100:00080000:0.0:1766099290.558846:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099290.558852:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00010000:00080000:3.0:1766099290.558858:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000adfcc04b x1851885432918848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099297 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099290.558862:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2880771 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432918720:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558886:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432918848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099290.558902:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000220e4534 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432918592:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099291.005775:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099292.030778:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099293.053848:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099294.091757:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099295.007924:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099295.007966:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099295.008037:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213120
00000100:00100000:1.0:1766099295.008107:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1602
00000100:00100000:1.0:1766099295.008117:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417213120:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099295.008216:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417213120:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 99us (304us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099295.008226:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1602
00000100:00100000:0.0:1766099295.009532:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099295.009743:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213184
00000100:00100000:3.0:1766099295.009818:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2360
00000100:00100000:3.0:1766099295.009826:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417213184:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099295.010038:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2880771 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417213184:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 211us (2071us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099295.010047:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2360
00000100:00100000:1.0:1766099295.010152:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213248
00000100:00100000:1.0:1766099295.010187:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1455
00000100:00100000:1.0:1766099295.010191:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417213248:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099295.011409:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099295.011518:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417213248:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1324us (1993us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099295.011533:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1455
00010000:00010000:0.0:1766099295.103194:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099295.685129:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099295.685137:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099295.685140:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099295.685148:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3799
00000100:00080000:2.0:1766099295.685152:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099295.685158:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099295.685177:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000000f47354 x1851885432919040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099295.685291:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000116a4423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432919168:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.685360:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099295.685393:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099295.685398:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca6dd622 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432918912:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.685446:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099295.685467:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5012ca9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918976:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.685518:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099295.685539:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099295.685544:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432919104:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.685565:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099295.685586:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432919040:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099295.685608:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099295.685639:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919168
00000100:00100000:2.0:1766099295.685675:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919040
00000100:00100000:2.0:1766099295.685692:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1065
00000100:00100000:2.0:1766099295.685698:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000411dc2e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432919168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.685746:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000411dc2e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432919168:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (387us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099295.685753:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1065
00000100:00100000:2.0:1766099295.685768:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1456
00000100:00100000:2.0:1766099295.685772:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432919040:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099295.685868:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099295.685898:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432919040:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (290us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099295.685904:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1456
00000100:00100000:2.0:1766099295.685922:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918976
00000100:00100000:2.0:1766099295.685941:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919104
00000100:00100000:2.0:1766099295.685957:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2361
00000100:00100000:2.0:1766099295.685960:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.686006:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432918976:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (487us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099295.686012:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2361
00000100:00100000:2.0:1766099295.686022:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2362
00000100:00100000:2.0:1766099295.686025:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003520602a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432919104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.686047:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003520602a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432919104:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (482us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099295.686052:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2362
00000100:00080000:2.0:1766099295.686084:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099295.686087:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099295.686091:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432919040:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099295.686118:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432919104:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099295.686126:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5012ca9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432918976:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766099295.686143:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000116a4423 x1851885432919168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099302 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099295.686151:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000116a4423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432919168:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099295.686434:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432918912
00000100:00100000:1.0:1766099295.686472:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1603
00000100:00100000:1.0:1766099295.686495:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432918912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099295.686534:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432918912:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (1088us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099295.686540:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1603
00000100:00100000:1.0:1766099295.686589:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca6dd622 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432918912:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099296.126897:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099297.154785:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099298.202059:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099299.261685:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099300.126532:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099300.126587:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099300.126632:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099300.126674:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213440
00000100:00100000:0.0:1766099300.126708:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1457
00000100:00100000:0.0:1766099300.126714:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfede6c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417213440:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099300.126810:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099300.126866:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfede6c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417213440:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 151us (234us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099300.126873:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1457
00000100:00100000:1.0:1766099300.128613:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213312
00000100:00100000:1.0:1766099300.128675:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1604
00000100:00100000:1.0:1766099300.128681:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417213312:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099300.128741:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417213312:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 60us (2155us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099300.128747:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1604
00000100:00100000:3.0:1766099300.129184:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213376
00000100:00100000:3.0:1766099300.129230:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2363
00000100:00100000:3.0:1766099300.129234:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062b9fb1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417213376:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099300.129277:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062b9fb1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417213376:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 41us (2753us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099300.129283:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2363
00010000:00010000:2.0:1766099300.285894:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099300.797997:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099300.798004:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099300.798007:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099300.798015:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3804
00000100:00100000:2.0:1766099300.798016:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432919232:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099300.798018:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099300.798023:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099300.798043:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000018198959 x1851885432919360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099300.798078:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432919424:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798080:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099300.798105:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099300.798109:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099300.798110:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432919488:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099300.798125:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099300.798128:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432919296:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798130:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099300.798146:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099300.798154:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432919360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798173:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099300.798174:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919424
00000100:00100000:2.0:1766099300.798196:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919488
00000100:00100000:1.0:1766099300.798207:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919296
00000100:00100000:2.0:1766099300.798228:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919360
00000100:00100000:2.0:1766099300.798247:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1066
00000100:00100000:2.0:1766099300.798252:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432919488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798296:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432919488:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (166us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099300.798304:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1066
00000100:00100000:2.0:1766099300.798318:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1458
00000100:00100000:2.0:1766099300.798321:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432919360:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099300.798345:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2364
00000100:00100000:1.0:1766099300.798349:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432919424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099300.798378:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432919424:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (269us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099300.798384:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2364
00000100:00100000:1.0:1766099300.798395:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2365
00000100:00100000:1.0:1766099300.798398:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a7d46da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432919296:12345-0@lo:400:kworker/u8:3.0
00010000:02020000:2.0:1766099300.798412:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099300.798417:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a7d46da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432919296:12345-0@lo:400:kworker/u8:3.0 Request processed in 19us (270us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099300.798421:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2365
00000100:00100000:2.0:1766099300.798436:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432919360:12345-0@lo:8:kworker/u8:3.0 Request processed in 114us (263us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099300.798442:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1458
00000100:00100000:1.0:1766099300.798461:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432919296:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766099300.798462:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099300.798465:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099300.798468:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432919360:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099300.798470:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432919424:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798510:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919232
00000100:00100000:2.0:1766099300.798530:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1605
00000100:00100000:2.0:1766099300.798534:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432919232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798561:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432919232:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (483us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099300.798567:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1605
00010000:00080000:2.0:1766099300.798595:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ce45f83d x1851885432919488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099307 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099300.798604:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432919488:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099300.798611:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432919232:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099301.309983:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099302.334630:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099303.360492:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099304.382114:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099305.249091:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099305.249139:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099305.249162:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213504
00000100:00100000:0.0:1766099305.249215:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099305.249225:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1606
00000100:00100000:2.0:1766099305.249233:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417213504:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099305.249313:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417213504:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 79us (229us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099305.249321:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1606
00000100:00100000:1.0:1766099305.249437:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213632
00000100:00100000:1.0:1766099305.249468:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1459
00000100:00100000:1.0:1766099305.249472:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000032cf85a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417213632:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099305.249587:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099305.249615:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000032cf85a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417213632:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 142us (477us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099305.249625:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1459
00000100:00100000:3.0:1766099305.249699:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213568
00000100:00100000:3.0:1766099305.250331:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2366
00000100:00100000:3.0:1766099305.250337:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008e61d6e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417213568:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099305.254018:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008e61d6e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417213568:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 3673us (4796us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099305.254048:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2366
00010000:00010000:0.0:1766099305.409522:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099305.922469:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099305.922501:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099305.922505:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099305.922514:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3809
00000100:00080000:0.0:1766099305.922517:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099305.922524:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099305.922545:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000adfcc04b x1851885432919680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099305.922591:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432919552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099305.922615:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432919808:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099305.922673:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099305.922676:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099305.922735:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432919744:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099305.923056:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432919680:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099305.923059:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000220e4534 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432919616:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099305.923092:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099305.923129:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099305.923135:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919808
00000100:00100000:3.0:1766099305.923175:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919616
00000100:00100000:2.0:1766099305.923180:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919680
00000100:00100000:2.0:1766099305.923205:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1067
00000100:00100000:2.0:1766099305.923211:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432919808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099305.923215:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2367
00000100:00100000:3.0:1766099305.923221:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093ed14f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432919616:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099305.923270:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432919808:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (598us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099305.923273:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093ed14f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432919616:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (145us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099305.923282:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1067
00000100:00100000:3.0:1766099305.923286:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2367
00000100:00100000:2.0:1766099305.923301:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1460
00000100:00100000:2.0:1766099305.923305:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c572263a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432919680:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099305.923382:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000220e4534 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432919616:0@lo:400:kworker/u8:2.0
00010000:02020000:2.0:1766099305.923405:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099305.923420:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919552
00000100:00100000:2.0:1766099305.923438:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c572263a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432919680:12345-0@lo:8:kworker/u8:2.0 Request processed in 130us (346us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099305.923446:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1460
00000100:00100000:3.0:1766099305.923450:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1607
00000100:00100000:3.0:1766099305.923456:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432919552:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099305.923504:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099305.923509:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099305.923513:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432919680:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099305.923523:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432919552:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (850us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099305.923531:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1607
00000100:00100000:2.0:1766099305.923550:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004aef0954 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432919552:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766099305.924970:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000096d55fda x1851885432919808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099312 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099305.924984:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432919808:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099305.925209:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099305.925296:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919744
00000100:00100000:2.0:1766099305.925338:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2368
00000100:00100000:2.0:1766099305.925343:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026f8c99c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432919744:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099305.925395:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026f8c99c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432919744:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (188us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099305.925402:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2368
00000100:00100000:1.0:1766099305.925458:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432919744:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099306.429692:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099307.453708:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099308.477724:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099309.502141:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099310.365781:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099310.365842:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099310.365896:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099310.366009:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213760
00000100:00100000:1.0:1766099310.366011:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213824
00000100:00100000:3.0:1766099310.366101:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2369
00000100:00100000:1.0:1766099310.366110:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1461
00000100:00100000:3.0:1766099310.366116:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096d55fda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417213760:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099310.366127:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417213824:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:3.0:1766099310.366222:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096d55fda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417213760:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 110us (453us total) trans 0 rc 0/0
00010000:02020000:1.0:1766099310.366234:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099310.366264:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2369
00000100:00100000:1.0:1766099310.366305:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417213824:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 183us (407us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099310.366315:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1461
00000100:00100000:2.0:1766099310.367521:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213696
00000100:00100000:2.0:1766099310.368147:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1608
00000100:00100000:2.0:1766099310.368156:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417213696:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099310.369119:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417213696:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 960us (3273us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099310.369130:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1608
00010000:00010000:0.0:1766099310.527859:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099311.040423:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099311.041105:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432919872:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099311.041178:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099311.041242:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919872
00000100:00100000:1.0:1766099311.041286:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1609
00000100:00100000:1.0:1766099311.041292:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432919872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099311.041339:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432919872:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (162us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099311.041346:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1609
00000100:00100000:1.0:1766099311.041388:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432919872:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099311.048753:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099311.048761:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099311.048764:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099311.048772:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3814
00000100:00080000:2.0:1766099311.048776:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099311.048781:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099311.048803:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ca6dd622 x1851885432920000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099311.049056:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca6dd622 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920000:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099311.049119:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099311.049159:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432920128:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099311.049181:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099311.049216:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920000
00000100:00100000:2.0:1766099311.049244:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920128
00000100:00100000:2.0:1766099311.049271:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068
00000100:00100000:2.0:1766099311.049276:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432920128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099311.049321:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432920128:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (139us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099311.049328:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068
00000100:00100000:2.0:1766099311.049343:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1462
00000100:00100000:2.0:1766099311.049347:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432920000:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099311.049437:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099311.049466:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432920000:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (349us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099311.049472:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1462
00010000:00080000:2.0:1766099311.049525:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a6ab8e56 x1851885432920128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099318 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099311.049536:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432920128:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099311.049560:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099311.049563:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099311.049566:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca6dd622 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920000:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099311.049655:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432920064:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099311.049686:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099311.049713:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099311.049719:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432919936:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099311.049739:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099311.049776:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920064
00000100:00100000:0.0:1766099311.049801:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432919936
00000100:00100000:0.0:1766099311.049816:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2370
00000100:00100000:0.0:1766099311.049820:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000772d840f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432920064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099311.049851:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000772d840f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432920064:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (165us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099311.049872:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2370
00000100:00100000:0.0:1766099311.049885:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2371
00000100:00100000:0.0:1766099311.049888:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006de96a83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432919936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099311.049911:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006de96a83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432919936:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (172us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099311.049917:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2371
00000100:00100000:0.0:1766099311.049944:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432919936:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099311.049952:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432920064:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099311.550264:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099312.574223:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099313.597770:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099314.621805:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099315.485605:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099315.485648:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099315.485768:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213952
00000100:00100000:2.0:1766099315.485823:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2372
00000100:00100000:2.0:1766099315.485838:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e44afcc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417213952:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099315.485908:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e44afcc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417213952:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 70us (260us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099315.485916:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2372
00000100:00100000:3.0:1766099315.486014:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417213888
00000100:00100000:3.0:1766099315.486050:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1610
00000100:00100000:3.0:1766099315.486055:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417213888:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099315.486082:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417213888:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 26us (490us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099315.486087:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1610
00000100:00100000:0.0:1766099315.486424:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099315.486566:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214016
00000100:00100000:2.0:1766099315.486596:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1463
00000100:00100000:2.0:1766099315.486600:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084b59448 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214016:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099315.486692:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099315.486736:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084b59448 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214016:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 135us (312us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099315.486742:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1463
00010000:00010000:2.0:1766099315.645728:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099316.158906:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099316.158913:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099316.158916:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099316.158923:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3819
00000100:00080000:2.0:1766099316.158926:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099316.158931:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099316.158951:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001e4d8abb x1851885432920320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099316.159006:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432920384:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099316.159064:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099316.159090:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099316.159095:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432920256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099316.159135:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099316.159160:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920320:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099316.159186:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099316.159224:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920384
00000100:00100000:2.0:1766099316.159261:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920256
00000100:00100000:2.0:1766099316.159280:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2373
00000100:00100000:2.0:1766099316.159286:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432920384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099316.159333:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432920384:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (269us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099316.159340:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2373
00000100:00100000:2.0:1766099316.159356:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2374
00000100:00100000:2.0:1766099316.159360:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068cf0967 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432920256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099316.159383:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068cf0967 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432920256:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (249us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099316.159388:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2374
00000100:00100000:2.0:1766099316.159434:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432920256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099316.159446:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca5e8fcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432920384:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099316.159545:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920320
00000100:00100000:3.0:1766099316.159578:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1464
00000100:00100000:3.0:1766099316.159581:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbe9c3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432920320:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099316.159669:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099316.159704:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbe9c3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432920320:12345-0@lo:8:kworker/u8:2.0 Request processed in 122us (518us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099316.159710:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1464
00000100:00100000:1.0:1766099316.159780:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432920192:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099316.159817:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099316.159863:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099316.159867:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432920448:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099316.159887:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099316.159909:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920448
00000100:00100000:1.0:1766099316.159934:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1069
00000100:00100000:1.0:1766099316.159937:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432920448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099316.159967:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432920448:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (79us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099316.159972:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1069
00010000:00080000:1.0:1766099316.159999:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000000f47354 x1851885432920448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099323 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099316.160007:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432920448:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099316.160070:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920192
00000100:00100000:3.0:1766099316.160102:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1611
00000100:00100000:3.0:1766099316.160106:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432920192:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099316.160139:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432920192:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (323us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099316.160144:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1611
00000100:00100000:1.0:1766099316.160563:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6ab8e56 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432920192:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099316.164330:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099316.164340:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099316.164347:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920320:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099316.674594:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099317.694662:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099318.717721:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099319.746213:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099320.605433:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099320.607305:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099320.607390:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214144
00000100:00100000:1.0:1766099320.607441:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2375
00000100:00100000:1.0:1766099320.607448:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de0c7c85 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417214144:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099320.607540:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de0c7c85 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417214144:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 91us (2114us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099320.607548:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2375
00000100:00100000:3.0:1766099320.609776:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214080
00000100:00100000:3.0:1766099320.609928:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1612
00000100:00100000:3.0:1766099320.609935:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417214080:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099320.610013:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417214080:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 77us (2715us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099320.610029:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1612
00000100:00100000:0.0:1766099320.610175:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099320.611230:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214208
00000100:00100000:3.0:1766099320.611531:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1465
00000100:00100000:3.0:1766099320.611538:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000399d91fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214208:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099320.611635:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099320.611703:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000399d91fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214208:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 164us (1529us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099320.611709:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1465
00010000:00010000:0.0:1766099320.765775:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099321.279380:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099321.279387:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099321.279390:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099321.279398:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3824
00000100:00080000:2.0:1766099321.279401:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099321.279406:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099321.279424:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004241b1eb x1851885432920640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099321.279466:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001169bcfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432920704:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.279543:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099321.279573:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099321.279592:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432920576:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.279616:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099321.279639:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004241b1eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920640:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099321.279662:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099321.279696:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920704
00000100:00100000:2.0:1766099321.279736:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920576
00000100:00100000:2.0:1766099321.279752:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2376
00000100:00100000:2.0:1766099321.279758:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000096a1d66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432920704:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.279832:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000096a1d66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432920704:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (289us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099321.279840:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2376
00000100:00100000:2.0:1766099321.279857:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2377
00000100:00100000:2.0:1766099321.279860:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aacc79c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432920576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.279884:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aacc79c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432920576:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (268us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099321.279890:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2377
00000100:00100000:2.0:1766099321.279908:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920640
00000100:00100000:2.0:1766099321.279926:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1466
00000100:00100000:2.0:1766099321.279930:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8301247 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432920640:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099321.280018:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099321.280063:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8301247 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432920640:12345-0@lo:8:kworker/u8:2.0 Request processed in 132us (400us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099321.280069:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1466
00000100:00100000:2.0:1766099321.280107:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432920576:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.280116:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001169bcfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432920704:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099321.280140:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099321.280143:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099321.280147:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004241b1eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920640:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099321.280611:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085cf7081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432920512:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099321.280653:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099321.280673:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099321.280678:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080c118b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432920768:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099321.280698:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099321.281380:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920768
00000100:00100000:0.0:1766099321.281424:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1070
00000100:00100000:0.0:1766099321.281429:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f11e586 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432920768:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099321.281507:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f11e586 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432920768:12345-0@lo:400:kworker/u8:2.0 Request processed in 77us (808us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099321.281514:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1070
00010000:00080000:1.0:1766099321.281737:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000080c118b3 x1851885432920768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099328 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099321.281747:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080c118b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432920768:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.282855:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920512
00000100:00100000:2.0:1766099321.282902:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1613
00000100:00100000:2.0:1766099321.282907:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432920512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099321.283110:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432920512:12345-0@lo:400:kworker/u8:2.0 Request processed in 201us (2456us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099321.283117:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1613
00000100:00100000:1.0:1766099321.283179:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085cf7081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432920512:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099321.790342:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099322.813684:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099323.837827:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099324.861720:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099325.732120:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099325.732175:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099325.732223:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099325.732321:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214400
00000100:00100000:3.0:1766099325.732384:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1467
00000100:00100000:3.0:1766099325.732392:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214400:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766099325.732536:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214336
00010000:02020000:3.0:1766099325.732545:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099325.732610:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2378
00000100:00100000:2.0:1766099325.732618:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417214336:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099325.732636:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214400:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 243us (411us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099325.732646:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1467
00000100:00100000:2.0:1766099325.732703:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417214336:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 82us (593us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099325.732716:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2378
00000100:00100000:1.0:1766099325.733644:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214272
00000100:00100000:1.0:1766099325.733702:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1614
00000100:00100000:1.0:1766099325.733708:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417214272:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099325.733787:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417214272:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 78us (1611us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099325.733796:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1614
00010000:00010000:0.0:1766099325.889226:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099326.400322:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099326.400330:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099326.400333:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099326.400341:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3829
00000100:00080000:1.0:1766099326.400345:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099326.400351:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099326.400372:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000018198959 x1851885432920960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099326.400461:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432920896:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.400553:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099326.400567:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b63cd4b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432921088:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.400583:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099326.400593:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432921024:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099326.400611:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099326.400636:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099326.400641:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099326.400644:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432920832:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.400675:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920960:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099326.400682:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099326.400710:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099326.400755:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920896
00000100:00100000:2.0:1766099326.400790:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921024
00000100:00100000:2.0:1766099326.400808:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2379
00000100:00100000:2.0:1766099326.400812:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004241b1eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432920896:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.400869:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004241b1eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432920896:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (318us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099326.400881:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2379
00000100:00100000:2.0:1766099326.400896:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2380
00000100:00100000:2.0:1766099326.400900:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001169bcfc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432921024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.400926:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001169bcfc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432921024:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (287us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099326.400932:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2380
00000100:00100000:3.0:1766099326.400942:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920832
00000100:00100000:2.0:1766099326.400975:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432921024:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099326.400976:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1615
00000100:00100000:3.0:1766099326.400984:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432920832:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.400992:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432920896:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099326.401093:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432920832:12345-0@lo:400:kworker/u8:3.0 Request processed in 109us (415us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099326.401101:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1615
00000100:00100000:3.0:1766099326.401140:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432920832:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099326.401317:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921088
00000100:00100000:2.0:1766099326.401340:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432920960
00000100:00100000:0.0:1766099326.401357:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1071
00000100:00100000:0.0:1766099326.401364:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007754fd34 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432921088:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099326.401411:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007754fd34 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432921088:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (800us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099326.401437:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1071
00000100:00100000:2.0:1766099326.401452:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1468
00000100:00100000:2.0:1766099326.401458:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094c05093 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432920960:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099326.401574:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:1.0:1766099326.402463:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b63cd4b7 x1851885432921088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099333 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099326.402490:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b63cd4b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432921088:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099326.402593:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094c05093 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432920960:12345-0@lo:8:kworker/u8:3.0 Request processed in 1134us (1885us total) trans 0 rc -19/-19
00000100:00080000:3.0:1766099326.402775:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099326.402793:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099326.402797:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432920960:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099326.402923:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1468
00010000:00010000:0.0:1766099326.909622:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099327.934581:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099328.957963:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099329.981792:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099330.845664:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099330.845743:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099330.845769:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214528
00000100:00100000:0.0:1766099330.845789:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099330.845852:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2381
00000100:00100000:1.0:1766099330.845861:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417214528:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099330.845966:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417214528:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 105us (315us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099330.845974:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2381
00000100:00100000:3.0:1766099330.853721:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214464
00000100:00100000:3.0:1766099330.853797:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1616
00000100:00100000:3.0:1766099330.853806:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417214464:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099330.853933:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417214464:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 126us (8192us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099330.853942:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1616
00000100:00100000:2.0:1766099330.858454:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214592
00000100:00100000:2.0:1766099330.858555:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1469
00000100:00100000:2.0:1766099330.858563:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214592:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099330.858671:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099330.858777:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214592:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 213us (12988us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099330.858786:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1469
00010000:00010000:2.0:1766099331.006986:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099331.517686:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099331.517695:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099331.517699:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099331.517718:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3834
00000100:00080000:1.0:1766099331.517721:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099331.517728:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099331.517753:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c025e697 x1851885432921280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099331.517809:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7e31e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432921408:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099331.517845:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c025e697 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432921280:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099331.518105:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b63cd4b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432921152:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099331.518326:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000529c8629 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432921216:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099331.518627:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099331.518719:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099331.518754:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432921344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099331.518787:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099331.518826:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921344
00000100:00100000:1.0:1766099331.518875:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2382
00000100:00100000:1.0:1766099331.518882:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090c751a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432921344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099331.518942:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090c751a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432921344:12345-0@lo:400:kworker/u8:3.0 Request processed in 60us (155us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099331.518951:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2382
00000100:00100000:1.0:1766099331.519008:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432921344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099331.519043:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921280
00000100:00100000:1.0:1766099331.519063:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921408
00000100:00100000:1.0:1766099331.519085:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1072
00000100:00100000:1.0:1766099331.519088:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432921408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099331.519123:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432921408:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (406us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099331.519130:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1072
00000100:00100000:1.0:1766099331.519141:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1470
00000100:00100000:1.0:1766099331.519145:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432921280:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099331.519242:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099331.519274:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432921280:12345-0@lo:8:kworker/u8:3.0 Request processed in 129us (651us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099331.519281:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1470
00010000:00080000:1.0:1766099331.519314:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d7e31e7d x1851885432921408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099338 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099331.519325:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7e31e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432921408:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099331.519443:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099331.519448:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099331.519452:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c025e697 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432921280:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099331.519755:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099331.519767:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099331.520631:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921152
00000100:00100000:3.0:1766099331.520680:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1617
00000100:00100000:3.0:1766099331.520686:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432921152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099331.520774:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432921152:12345-0@lo:400:kworker/u8:3.0 Request processed in 87us (1020us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099331.520780:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1617
00000100:00100000:2.0:1766099331.521039:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b63cd4b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432921152:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099331.521281:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921216
00000100:00100000:0.0:1766099331.522262:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2383
00000100:00100000:0.0:1766099331.522272:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084b59448 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432921216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099331.522378:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084b59448 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432921216:12345-0@lo:400:kworker/u8:3.0 Request processed in 108us (2613us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099331.522385:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2383
00000100:00100000:1.0:1766099331.523270:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000529c8629 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432921216:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099332.029974:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099333.053689:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099334.078103:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099335.101765:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099335.964993:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099335.965062:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099335.965107:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099335.965196:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214656
00000100:00100000:2.0:1766099335.965268:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1618
00000100:00100000:2.0:1766099335.965276:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417214656:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099335.965376:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417214656:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 100us (314us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099335.965384:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1618
00000100:00100000:3.0:1766099335.965698:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214784
00000100:00100000:3.0:1766099335.965734:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1471
00000100:00100000:3.0:1766099335.965739:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214784:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099335.965841:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099335.965899:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417214784:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 158us (792us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099335.965907:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1471
00000100:00100000:1.0:1766099335.965995:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214720
00000100:00100000:1.0:1766099335.966041:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2384
00000100:00100000:1.0:1766099335.966046:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000399d91fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417214720:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099335.966082:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000399d91fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417214720:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 35us (1101us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099335.966088:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2384
00010000:00010000:0.0:1766099336.129427:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099336.637717:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099336.637727:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099336.637731:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099336.637742:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3839
00000100:00080000:1.0:1766099336.637745:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099336.637753:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099336.637779:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f68a282c x1851885432921600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099336.637829:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c44ad7a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432921664:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099336.637911:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099336.637942:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099336.637947:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7e31e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432921536:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099336.637968:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099336.638004:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921664
00000100:00100000:1.0:1766099336.638051:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921536
00000100:00100000:1.0:1766099336.638074:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2385
00000100:00100000:1.0:1766099336.638080:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080791d40 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432921664:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099336.638132:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080791d40 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432921664:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (223us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099336.638150:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2385
00000100:00100000:1.0:1766099336.638172:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2386
00000100:00100000:1.0:1766099336.638176:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce4ad370 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432921536:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099336.638200:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce4ad370 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432921536:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (232us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099336.638205:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2386
00000100:00100000:1.0:1766099336.638252:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7e31e7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432921536:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099336.638260:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c44ad7a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432921664:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099336.638453:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000529c8629 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432921472:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099336.638519:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099336.638539:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099336.638543:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432921728:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099336.638569:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099336.638601:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432921600:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099336.638624:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099336.638653:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921728
00000100:00100000:3.0:1766099336.638696:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921600
00000100:00100000:3.0:1766099336.638715:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1073
00000100:00100000:3.0:1766099336.638718:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432921728:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099336.638751:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432921728:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (182us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099336.638757:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1073
00000100:00100000:3.0:1766099336.638769:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1472
00000100:00100000:3.0:1766099336.638773:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432921600:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099336.638878:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099336.638906:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432921600:12345-0@lo:8:kworker/u8:3.0 Request processed in 132us (282us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099336.638912:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1472
00010000:00080000:3.0:1766099336.638937:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000037d37a0f x1851885432921728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099343 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099336.638969:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432921728:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099336.638994:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099336.638998:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099336.639001:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432921600:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099336.639115:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921472
00000100:00100000:0.0:1766099336.639158:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1619
00000100:00100000:0.0:1766099336.639161:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432921472:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099336.639206:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432921472:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (687us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099336.639212:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1619
00000100:00100000:1.0:1766099336.639283:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000529c8629 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432921472:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099337.149943:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099338.173747:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099339.198368:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099340.221725:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099341.088474:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099341.088563:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099341.088670:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214912
00000100:00100000:3.0:1766099341.089604:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214848
00000100:00100000:3.0:1766099341.089655:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1620
00000100:00100000:3.0:1766099341.089663:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417214848:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099341.089696:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2387
00000100:00100000:0.0:1766099341.089710:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000529c8629 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417214912:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099341.089853:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099341.089907:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417214976
00000100:00100000:3.0:1766099341.089926:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417214848:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 262us (1465us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099341.089939:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1620
00000100:00100000:2.0:1766099341.089940:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1473
00000100:00100000:2.0:1766099341.089947:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417214976:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099341.090048:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099341.090082:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417214976:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 134us (229us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099341.090091:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1473
00000100:00100000:0.0:1766099341.090141:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000529c8629 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417214912:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 432us (1577us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099341.090149:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2387
00010000:00010000:0.0:1766099341.249721:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099341.758031:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099341.758041:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:3.0:1766099341.758041:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e7c243d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432921792:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099341.758047:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099341.758055:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3845
00000100:00080000:0.0:1766099341.758058:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099341.758065:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099341.758092:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000492edb9a x1851885432921920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099341.758147:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099341.758160:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1968617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432921856:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099341.758187:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099341.758196:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a851b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432922048:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758203:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099341.758229:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099341.758234:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099341.758237:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a3dc47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432921984:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758265:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099341.758284:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922048
00000100:00100000:0.0:1766099341.758298:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921856
00000100:00100000:0.0:1766099341.758337:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921984
00000100:00100000:3.0:1766099341.758346:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1074
00000100:00100000:3.0:1766099341.758355:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432922048:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758360:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2388
00000100:00100000:0.0:1766099341.758366:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de379e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432921856:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758424:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de379e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432921856:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (221us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099341.758429:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432922048:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (193us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099341.758436:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2388
00000100:00100000:3.0:1766099341.758440:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1074
00000100:00100000:0.0:1766099341.758466:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2389
00000100:00100000:3.0:1766099341.758469:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921792
00000100:00100000:0.0:1766099341.758473:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eaec260 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432921984:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099341.758519:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1621
00000100:00100000:3.0:1766099341.758525:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432921792:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758530:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eaec260 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432921984:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (263us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099341.758537:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2389
00000100:00100000:3.0:1766099341.758573:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432921792:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (428us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099341.758580:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1621
00000100:00100000:0.0:1766099341.758582:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a3dc47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432921984:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758598:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1968617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432921856:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099341.758614:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432921920:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099341.758650:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099341.758672:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432921920
00010000:00080000:3.0:1766099341.758687:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a8a851b9 x1851885432922048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099348 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099341.758696:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1474
00000100:00100000:3.0:1766099341.758703:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a851b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432922048:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099341.758705:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432921920:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099341.758714:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e7c243d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432921792:0@lo:400:kworker/u8:2.0
00010000:02020000:0.0:1766099341.758799:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099341.758846:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432921920:12345-0@lo:8:kworker/u8:2.0 Request processed in 142us (196us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099341.758851:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1474
00000100:00080000:3.0:1766099341.759077:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099341.759083:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099341.759086:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432921920:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099342.270286:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099343.293787:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099344.318728:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099345.342818:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099346.204757:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099346.204816:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099346.204864:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099346.204983:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215168
00000100:00100000:0.0:1766099346.205051:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1475
00000100:00100000:0.0:1766099346.205059:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7e31e7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417215168:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099346.205166:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099346.205259:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7e31e7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417215168:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 198us (514us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099346.205267:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1475
00000100:00100000:3.0:1766099346.205593:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215040
00000100:00100000:3.0:1766099346.205643:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1622
00000100:00100000:3.0:1766099346.205648:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417215040:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099346.205701:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417215040:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 52us (837us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099346.205709:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1622
00000100:00100000:2.0:1766099346.206044:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215104
00000100:00100000:2.0:1766099346.206913:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2390
00000100:00100000:2.0:1766099346.206922:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1ac28d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417215104:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099346.206998:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1ac28d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417215104:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 76us (2182us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099346.207009:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2390
00010000:00010000:2.0:1766099346.371751:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099346.878445:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099346.878455:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099346.878459:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099346.878470:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3850
00000100:00080000:1.0:1766099346.878475:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099346.878504:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099346.878513:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099346.878528:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432922112:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099346.878531:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b63cd4b7 x1851885432922240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099346.878579:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c025e697 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432922304:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099346.878586:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432922176:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099346.878846:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099346.879034:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099346.879051:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099346.879113:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922304
00000100:00100000:0.0:1766099346.879167:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922176
00000100:00100000:0.0:1766099346.879192:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2391
00000100:00100000:0.0:1766099346.879198:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432922304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099346.879255:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c15c4086 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432922304:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (410us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099346.879263:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2391
00000100:00100000:0.0:1766099346.879284:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2392
00000100:00100000:0.0:1766099346.879287:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1968617 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432922176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099346.879317:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1968617 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432922176:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (267us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099346.879323:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2392
00000100:00100000:2.0:1766099346.879625:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cad6b0f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432922368:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099346.879656:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099346.879690:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922368
00000100:00100000:2.0:1766099346.879720:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1075
00000100:00100000:2.0:1766099346.879725:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e6f9e94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432922368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099346.879762:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e6f9e94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432922368:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (105us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099346.879769:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1075
00000100:00100000:0.0:1766099346.879798:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d37a0f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432922176:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766099346.880279:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cad6b0f4 x1851885432922368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099353 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099346.880372:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922112
00000100:00100000:0.0:1766099346.880542:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c025e697 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432922304:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099346.880542:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cad6b0f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432922368:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099346.880624:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1623
00000100:00100000:1.0:1766099346.880630:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432922112:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099346.880699:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432922112:12345-0@lo:400:kworker/u8:3.0 Request processed in 69us (1665us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099346.880706:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1623
00000100:00100000:2.0:1766099346.880786:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f68a282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432922112:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099346.881235:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b63cd4b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432922240:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099346.881277:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099346.881585:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922240
00000100:00100000:2.0:1766099346.881625:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1476
00000100:00100000:2.0:1766099346.881629:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432922240:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099346.881727:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099346.881773:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432922240:12345-0@lo:8:kworker/u8:3.0 Request processed in 143us (496us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099346.881779:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1476
00000100:00080000:3.0:1766099346.882760:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099346.882769:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099346.882775:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b63cd4b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432922240:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099347.391073:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099348.413831:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099349.437861:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099350.462284:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099351.328517:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099351.328584:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099351.328708:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215296
00000100:00100000:0.0:1766099351.328780:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2393
00000100:00100000:0.0:1766099351.328788:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417215296:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099351.328884:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde33f60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417215296:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 96us (407us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099351.328892:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2393
00000100:00100000:1.0:1766099351.329275:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099351.329821:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215232
00000100:00100000:3.0:1766099351.329881:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1624
00000100:00100000:3.0:1766099351.329887:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417215232:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099351.329956:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417215232:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 68us (1373us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099351.329964:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1624
00000100:00100000:0.0:1766099351.331182:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215360
00000100:00100000:0.0:1766099351.331246:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1477
00000100:00100000:0.0:1766099351.331252:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417215360:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099351.331403:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099351.331524:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417215360:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 270us (2250us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099351.331532:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1477
00010000:00010000:0.0:1766099351.491619:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099351.998835:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099351.998845:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099351.998850:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099351.998861:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3855
00000100:00080000:0.0:1766099351.998865:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099351.998873:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099351.998899:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001d84c111 x1851885432922560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099351.998951:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067440601 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432922496:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099351.999026:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099351.999057:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099351.999061:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432922624:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099351.999080:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099351.999116:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922496
00000100:00100000:0.0:1766099351.999157:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922624
00000100:00100000:0.0:1766099351.999175:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2394
00000100:00100000:0.0:1766099351.999180:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432922496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099351.999227:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432922496:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (202us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099351.999234:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2394
00000100:00100000:0.0:1766099351.999252:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2395
00000100:00100000:0.0:1766099351.999255:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432922624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099351.999272:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432922624:12345-0@lo:400:kworker/u8:3.0 Request processed in 16us (192us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099351.999277:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2395
00000100:00100000:0.0:1766099351.999317:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432922624:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099351.999325:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067440601 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432922496:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099351.999421:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432922560:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099351.999459:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099351.999647:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bb18fa0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432922688:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099351.999680:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099351.999701:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099351.999706:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a3dc47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432922432:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099351.999732:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099351.999758:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922560
00000100:00100000:2.0:1766099351.999779:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922688
00000100:00100000:2.0:1766099351.999800:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076
00000100:00100000:2.0:1766099351.999804:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000083021c1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432922688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099351.999834:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000083021c1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432922688:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (155us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099351.999841:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076
00000100:00100000:2.0:1766099351.999851:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1478
00000100:00100000:2.0:1766099351.999855:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432922560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099351.999953:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099352.000110:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432922560:12345-0@lo:8:kworker/u8:3.0 Request processed in 254us (650us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099352.000117:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1478
00010000:00080000:2.0:1766099352.000151:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001bb18fa0 x1851885432922688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099358 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099352.000161:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bb18fa0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432922688:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766099352.000182:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099352.000186:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099352.000189:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432922560:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099352.000423:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922432
00000100:00100000:1.0:1766099352.000465:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1625
00000100:00100000:1.0:1766099352.000469:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432922432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099352.000546:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432922432:12345-0@lo:400:kworker/u8:3.0 Request processed in 76us (813us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099352.000552:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1625
00000100:00100000:2.0:1766099352.007265:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a3dc47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432922432:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099352.509977:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099353.533711:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099354.568959:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099355.584786:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099356.444684:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099356.445124:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099356.445288:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215424
00000100:00100000:3.0:1766099356.445367:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1626
00000100:00100000:3.0:1766099356.445375:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417215424:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099356.445470:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417215424:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (799us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099356.445535:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1626
00000100:00100000:1.0:1766099356.447179:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099356.447317:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215552
00000100:00100000:0.0:1766099356.447365:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1479
00000100:00100000:0.0:1766099356.447373:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b84bdab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417215552:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099356.447493:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099356.447589:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b84bdab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417215552:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 214us (2465us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099356.447608:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1479
00000100:00100000:2.0:1766099356.448007:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215488
00000100:00100000:2.0:1766099356.448059:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2396
00000100:00100000:2.0:1766099356.448065:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417215488:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099356.448115:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417215488:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 49us (945us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099356.448122:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2396
00010000:00010000:0.0:1766099356.605838:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099357.120123:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099357.120132:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099357.120137:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099357.120147:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3860
00000100:00080000:3.0:1766099357.120151:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766099357.120158:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099357.120167:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099357.120182:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432922752:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099357.120182:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b2cdda05 x1851885432922880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099357.120230:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8b9cdf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432922944:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099357.120666:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2cdda05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432922880:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099357.121045:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432922816:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099357.121141:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099357.121162:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099357.121181:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048d710c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432923008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099357.121210:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922944
00000100:00100000:1.0:1766099357.121226:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099357.121259:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922752
00000100:00100000:3.0:1766099357.121263:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2397
00000100:00100000:3.0:1766099357.121270:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ccf9eed1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432922944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099357.121282:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1627
00000100:00100000:1.0:1766099357.121285:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432922752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099357.121323:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432922752:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (183us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099357.121323:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ccf9eed1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432922944:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (161us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099357.121332:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1627
00000100:00100000:3.0:1766099357.121332:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2397
00000100:00100000:3.0:1766099357.121361:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923008
00000100:00100000:3.0:1766099357.121385:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1077
00000100:00100000:3.0:1766099357.121388:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006301b41a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432923008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099357.121395:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432922752:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099357.121415:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006301b41a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432923008:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (189us total) trans 0 rc 0/0
00010000:00080000:1.0:1766099357.121417:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000048d710c3 x1851885432923008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099364 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099357.121421:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1077
00000100:00100000:1.0:1766099357.121428:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048d710c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432923008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099357.121443:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8b9cdf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432922944:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099357.121596:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099357.121730:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922880
00000100:00100000:3.0:1766099357.121759:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1480
00000100:00100000:3.0:1766099357.121763:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfa9a69 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432922880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099357.121915:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099357.121961:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfa9a69 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432922880:12345-0@lo:8:kworker/u8:2.0 Request processed in 197us (366us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099357.121968:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1480
00000100:00100000:0.0:1766099357.122087:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766099357.122136:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099357.122142:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099357.122146:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2cdda05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432922880:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099357.122168:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432922816
00000100:00100000:3.0:1766099357.122201:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2398
00000100:00100000:3.0:1766099357.122206:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067440601 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432922816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099357.122257:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067440601 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432922816:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (171us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099357.122439:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432922816:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099357.122627:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2398
00010000:00010000:2.0:1766099357.630439:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099358.654786:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099359.692644:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099360.703030:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099361.564452:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099361.566229:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099361.566268:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099361.566380:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215680
00000100:00100000:1.0:1766099361.566432:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2399
00000100:00100000:1.0:1766099361.566440:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8b9cdf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417215680:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099361.566546:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8b9cdf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417215680:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 105us (332us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099361.566553:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2399
00000100:00100000:0.0:1766099361.567008:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215616
00000100:00100000:0.0:1766099361.567055:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1628
00000100:00100000:0.0:1766099361.567060:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417215616:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099361.567093:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417215616:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 32us (2651us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099361.567100:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1628
00000100:00100000:2.0:1766099361.568755:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215744
00000100:00100000:2.0:1766099361.568800:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1481
00000100:00100000:2.0:1766099361.568809:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417215744:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099361.568902:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099361.568945:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417215744:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 136us (2678us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099361.568951:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1481
00010000:00010000:2.0:1766099361.727245:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099362.241664:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099362.241671:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099362.241674:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099362.241681:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3865
00000100:00080000:0.0:1766099362.241684:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099362.241690:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099362.241712:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000078a3dc47 x1851885432923200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099362.241763:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a3dc47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432923200:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099362.241826:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099362.241878:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923200
00000100:00100000:0.0:1766099362.241909:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1482
00000100:00100000:0.0:1766099362.241914:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432923200:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099362.242003:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099362.242041:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432923200:12345-0@lo:8:kworker/u8:3.0 Request processed in 126us (215us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099362.242047:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1482
00000100:00080000:0.0:1766099362.242085:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099362.242088:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099362.242091:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a3dc47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432923200:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242160:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de717044 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432923136:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242191:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099362.242214:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099362.242219:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432923264:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242235:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099362.242263:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923136
00000100:00100000:1.0:1766099362.242286:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923264
00000100:00100000:1.0:1766099362.242299:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2400
00000100:00100000:1.0:1766099362.242303:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000663ddca5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432923136:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242334:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000663ddca5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432923136:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (143us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099362.242339:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2400
00000100:00100000:1.0:1766099362.242350:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2401
00000100:00100000:1.0:1766099362.242352:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421476e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432923264:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242368:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421476e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432923264:12345-0@lo:400:kworker/u8:3.0 Request processed in 16us (134us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099362.242373:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2401
00000100:00100000:1.0:1766099362.242389:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432923072:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242410:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099362.242421:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099362.242424:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bb18fa0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432923328:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242440:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099362.242460:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432923264:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099362.242468:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de717044 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432923136:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099362.242606:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923328
00000100:00100000:2.0:1766099362.242633:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1078
00000100:00100000:2.0:1766099362.242637:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432923328:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099362.242675:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432923328:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (234us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099362.242681:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1078
00010000:00080000:2.0:1766099362.242708:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001bb18fa0 x1851885432923328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099369 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099362.242718:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bb18fa0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432923328:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099362.242813:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923072
00000100:00100000:0.0:1766099362.242837:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1629
00000100:00100000:0.0:1766099362.242841:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432923072:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099362.242872:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432923072:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (462us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099362.242878:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1629
00000100:00100000:2.0:1766099362.242960:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432923072:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099362.749919:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099363.782632:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099364.797737:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099365.823833:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099366.685547:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099366.685669:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099366.685786:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215872
00000100:00100000:2.0:1766099366.685862:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2402
00000100:00100000:2.0:1766099366.685870:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eea6599 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417215872:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099366.686411:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215808
00000100:00100000:0.0:1766099366.686461:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1630
00000100:00100000:0.0:1766099366.686466:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417215808:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099366.687100:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417215808:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 630us (1621us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099366.687100:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eea6599 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417215872:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1227us (1429us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099366.687111:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1630
00000100:00100000:2.0:1766099366.687111:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2402
00000100:00100000:3.0:1766099366.687626:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099366.687916:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417215936
00000100:00100000:1.0:1766099366.687957:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1483
00000100:00100000:1.0:1766099366.687963:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2cdda05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417215936:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099366.688077:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099366.688146:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2cdda05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417215936:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 182us (522us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099366.688153:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1483
00010000:00010000:2.0:1766099366.848554:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099367.358294:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099367.358304:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099367.358308:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099367.358319:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3870
00000100:00080000:1.0:1766099367.358324:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099367.358331:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099367.358358:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000413b47c6 x1851885432923520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099367.358639:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432923648:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099367.358730:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099367.358764:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099367.358769:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432923392:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099367.358802:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099367.358827:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbeb8a65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432923456:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099367.358856:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099367.358877:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099367.358882:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000022d6139c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432923584:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099367.358903:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099367.358926:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432923520:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099367.358948:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099367.358987:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923648
00000100:00100000:1.0:1766099367.359039:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923520
00000100:00100000:1.0:1766099367.359060:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1079
00000100:00100000:1.0:1766099367.359067:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432923648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099367.359120:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432923648:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (393us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099367.359128:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1079
00000100:00100000:1.0:1766099367.359146:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1484
00000100:00100000:1.0:1766099367.359149:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432923520:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099367.359193:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923456
00000100:00100000:0.0:1766099367.359221:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923584
00000100:00100000:0.0:1766099367.359237:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2403
00000100:00100000:0.0:1766099367.359241:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000769cb361 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432923456:12345-0@lo:400:kworker/u8:3.0
00010000:02020000:1.0:1766099367.359248:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099367.359277:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000769cb361 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432923456:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (421us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099367.359277:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432923520:12345-0@lo:8:kworker/u8:3.0 Request processed in 127us (329us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099367.359284:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2403
00000100:00100000:1.0:1766099367.359284:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1484
00000100:00100000:0.0:1766099367.359299:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2404
00000100:00100000:0.0:1766099367.359302:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071acbd6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432923584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099367.359326:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071acbd6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432923584:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (423us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099367.359331:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2404
00000100:00080000:1.0:1766099367.359337:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099367.359341:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099367.359345:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432923520:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099367.359373:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000022d6139c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432923584:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099367.359376:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923392
00000100:00100000:1.0:1766099367.359382:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbeb8a65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432923456:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099367.359400:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1631
00000100:00100000:0.0:1766099367.359414:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432923392:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099367.359453:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432923392:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (651us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099367.359665:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432923392:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099367.359685:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000de913b58 x1851885432923648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099374 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099367.359694:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432923648:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099367.361740:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1631
00010000:00010000:2.0:1766099367.870738:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099368.893914:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099369.917714:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099370.942197:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099371.805007:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099371.805056:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099371.805238:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216064
00000100:00100000:1.0:1766099371.805302:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2405
00000100:00100000:1.0:1766099371.805309:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d710c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417216064:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099371.805404:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d710c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417216064:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (348us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099371.805420:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2405
00000100:00100000:0.0:1766099371.805593:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216000
00000100:00100000:0.0:1766099371.805639:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1632
00000100:00100000:0.0:1766099371.805644:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216000:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099371.805673:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216000:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 29us (673us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099371.805679:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1632
00000100:00100000:2.0:1766099371.811586:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099371.811673:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216128
00000100:00100000:1.0:1766099371.811725:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1485
00000100:00100000:1.0:1766099371.811732:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007acd880d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417216128:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099371.811838:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099371.811913:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007acd880d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417216128:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 180us (333us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099371.811922:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1485
00010000:00010000:0.0:1766099371.968158:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099372.477668:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099372.477675:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099372.477678:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099372.477686:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3875
00000100:00080000:1.0:1766099372.477690:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099372.477694:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099372.477713:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000413b47c6 x1851885432923840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099372.477775:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000022d6139c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432923776:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099372.477829:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099372.477856:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099372.477861:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000801bfca3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432923904:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099372.477880:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099372.477901:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432923840:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099372.477930:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099372.477965:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923776
00000100:00100000:1.0:1766099372.478002:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923904
00000100:00100000:1.0:1766099372.478019:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2406
00000100:00100000:1.0:1766099372.478024:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000817fb53b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432923776:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099372.478198:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000199cfba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432923968:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099372.478225:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099372.478245:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099372.478250:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbeb8a65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432923712:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099372.478275:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099372.478302:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923840
00000100:00100000:2.0:1766099372.478314:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923712
00000100:00100000:3.0:1766099372.478320:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432923968
00000100:00100000:2.0:1766099372.478336:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1633
00000100:00100000:3.0:1766099372.478336:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080
00000100:00100000:2.0:1766099372.478340:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432923712:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099372.478340:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432923968:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099372.478383:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432923968:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (151us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099372.478390:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080
00000100:00100000:3.0:1766099372.478433:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1486
00000100:00100000:3.0:1766099372.478437:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432923840:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099372.478440:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2407
00000100:00100000:0.0:1766099372.478444:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432923904:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099372.478496:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432923904:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (615us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099372.478502:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2407
00010000:02020000:3.0:1766099372.478550:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099372.478555:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000801bfca3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432923904:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099372.478580:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432923840:12345-0@lo:8:kworker/u8:3.0 Request processed in 141us (650us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099372.478586:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1486
00000100:00080000:0.0:1766099372.478830:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099372.478835:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099372.478838:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432923840:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099372.481140:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000817fb53b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432923776:12345-0@lo:400:kworker/u8:3.0 Request processed in 3108us (3304us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099372.481154:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2406
00000100:00100000:0.0:1766099372.482307:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000022d6139c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432923776:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099372.482434:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432923712:12345-0@lo:400:kworker/u8:3.0 Request processed in 4092us (4157us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099372.482445:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1633
00000100:00100000:3.0:1766099372.482640:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbeb8a65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432923712:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099372.482655:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000199cfba3 x1851885432923968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099379 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099372.482664:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000199cfba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432923968:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099372.992303:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099374.015095:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099375.040652:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099376.061852:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099376.924592:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099376.927194:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216192
00000100:00100000:2.0:1766099376.927284:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1634
00000100:00100000:2.0:1766099376.927292:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417216192:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099376.927406:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417216192:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 114us (2828us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099376.927413:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1634
00000100:00100000:3.0:1766099376.928655:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099376.931748:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099376.931982:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216256
00000100:00100000:2.0:1766099376.932069:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2408
00000100:00100000:2.0:1766099376.932077:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417216256:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099376.932178:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417216256:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 101us (3530us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099376.932187:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2408
00000100:00100000:0.0:1766099376.932270:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216320
00000100:00100000:0.0:1766099376.932305:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1487
00000100:00100000:0.0:1766099376.932309:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417216320:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099376.932415:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099376.932453:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417216320:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 143us (715us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099376.932459:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1487
00010000:00010000:2.0:1766099377.087337:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099377.597971:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099377.597980:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099377.597985:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099377.597996:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3880
00000100:00080000:3.0:1766099377.598000:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766099377.598006:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099377.598028:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005a16d6f1 x1851885432924160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099377.598075:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432924224:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598155:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099377.598189:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099377.598194:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432924096:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598212:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099377.598247:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924224
00000100:00100000:3.0:1766099377.598292:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924096
00000100:00100000:3.0:1766099377.598309:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2409
00000100:00100000:3.0:1766099377.598315:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc100191 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432924224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598360:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc100191 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432924224:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (207us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099377.598379:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2409
00000100:00100000:3.0:1766099377.598397:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2410
00000100:00100000:3.0:1766099377.598400:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009296b030 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432924096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598420:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009296b030 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432924096:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (209us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099377.598425:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2410
00000100:00100000:3.0:1766099377.598467:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432924096:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598500:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432924224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099377.598590:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432924032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099377.598627:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099377.598655:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099377.598659:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be292ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432924288:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099377.598681:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099377.598703:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924288
00000100:00100000:2.0:1766099377.598728:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1081
00000100:00100000:2.0:1766099377.598731:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d96f6de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432924288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099377.598759:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d96f6de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432924288:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (78us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099377.598764:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1081
00010000:00080000:2.0:1766099377.598784:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000be292ac7 x1851885432924288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099384 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099377.598792:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be292ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432924288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598854:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924032
00000100:00100000:3.0:1766099377.598883:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1635
00000100:00100000:3.0:1766099377.598887:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432924032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099377.598917:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432924032:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (290us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099377.598923:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1635
00000100:00100000:3.0:1766099377.598949:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432924032:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099377.599080:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432924160:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099377.599111:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099377.599719:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924160
00000100:00100000:3.0:1766099377.599746:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1488
00000100:00100000:3.0:1766099377.599750:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a3dc47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432924160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099377.599847:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099377.599884:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a3dc47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432924160:12345-0@lo:8:kworker/u8:3.0 Request processed in 134us (774us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099377.599890:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1488
00000100:00080000:0.0:1766099377.600303:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099377.600310:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099377.600315:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432924160:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099378.109761:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099379.135542:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099380.160559:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099381.182147:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099382.044675:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099382.044719:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099382.044763:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099382.044796:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216512
00000100:00100000:2.0:1766099382.044835:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1489
00000100:00100000:2.0:1766099382.044841:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417216512:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099382.044938:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099382.045008:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216448
00000100:00100000:2.0:1766099382.045010:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417216512:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 167us (291us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099382.045016:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1489
00000100:00100000:0.0:1766099382.045040:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2411
00000100:00100000:0.0:1766099382.045044:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417216448:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099382.045089:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417216448:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 43us (421us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099382.045125:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216384
00000100:00100000:3.0:1766099382.045151:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1636
00000100:00100000:3.0:1766099382.045155:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417216384:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099382.045185:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417216384:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 28us (423us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099382.045191:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1636
00000100:00100000:0.0:1766099382.045647:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2411
00010000:00010000:2.0:1766099382.205803:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099382.718597:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099382.718604:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099382.718608:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099382.718615:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3886
00000100:00080000:2.0:1766099382.718619:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099382.718624:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099382.718643:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ee68381b x1851885432924480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099382.718688:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee68381b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432924480:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099382.718746:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099382.718795:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924480
00000100:00100000:2.0:1766099382.718826:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1490
00000100:00100000:2.0:1766099382.718832:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f745127a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432924480:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099382.718929:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099382.718967:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f745127a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432924480:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (222us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099382.718975:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1490
00000100:00080000:2.0:1766099382.719010:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099382.719013:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099382.719016:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee68381b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432924480:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099382.719083:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c023a180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432924608:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099382.719110:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099382.719133:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099382.719138:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd789e54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432924352:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099382.719160:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099382.719180:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924608
00000100:00100000:1.0:1766099382.719203:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1082
00000100:00100000:1.0:1766099382.719206:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432924608:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099382.719238:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432924608:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (128us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099382.719244:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1082
00010000:00080000:1.0:1766099382.719262:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c023a180 x1851885432924608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099389 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099382.719271:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c023a180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432924608:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099382.719338:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061109c96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432924416:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099382.719371:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099382.719394:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099382.719398:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c0ab926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432924544:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099382.719416:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099382.719442:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924416
00000100:00100000:0.0:1766099382.719460:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924544
00000100:00100000:0.0:1766099382.719472:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2412
00000100:00100000:0.0:1766099382.719497:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e7c243d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432924416:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099382.719524:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e7c243d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432924416:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (153us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099382.719529:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2412
00000100:00100000:0.0:1766099382.719540:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2413
00000100:00100000:0.0:1766099382.719543:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432924544:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099382.719559:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432924544:12345-0@lo:400:kworker/u8:2.0 Request processed in 15us (142us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099382.719563:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2413
00000100:00100000:0.0:1766099382.719579:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c0ab926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432924544:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099382.719586:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061109c96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432924416:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099382.719659:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924352
00000100:00100000:3.0:1766099382.719693:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1637
00000100:00100000:3.0:1766099382.719697:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432924352:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099382.719730:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432924352:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (570us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099382.719736:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1637
00000100:00100000:3.0:1766099382.719766:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd789e54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432924352:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099383.230338:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099384.254584:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099385.278264:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099386.303378:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099387.168947:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099387.168991:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099387.169063:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216576
00000100:00100000:1.0:1766099387.169066:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099387.169107:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1638
00000100:00100000:2.0:1766099387.169114:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216576:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099387.169573:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216704
00000100:00100000:3.0:1766099387.169598:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1491
00000100:00100000:3.0:1766099387.169602:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004bd75cc9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417216704:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099387.169692:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099387.169761:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216640
00000100:00100000:0.0:1766099387.169787:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2414
00000100:00100000:0.0:1766099387.169791:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3ebfac1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417216640:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099387.169839:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004bd75cc9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417216704:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 236us (773us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099387.169840:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216576:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 724us (847us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099387.169848:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1638
00000100:00100000:3.0:1766099387.169848:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1491
00000100:00100000:0.0:1766099387.169908:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3ebfac1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417216640:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 113us (964us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099387.169915:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2414
00010000:00010000:0.0:1766099387.326699:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099387.838562:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099387.838570:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099387.838573:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099387.838580:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3891
00000100:00080000:0.0:1766099387.838583:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099387.838589:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099387.838608:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001bb18fa0 x1851885432924800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099387.838650:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432924736:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099387.838712:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099387.838739:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099387.838745:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de717044 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432924864:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099387.838767:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099387.838789:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bb18fa0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432924800:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099387.838814:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099387.838845:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924736
00000100:00100000:1.0:1766099387.838948:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432924928:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099387.838972:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099387.838985:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099387.838989:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432924672:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099387.839007:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099387.839029:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924672
00000100:00100000:1.0:1766099387.839073:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1639
00000100:00100000:1.0:1766099387.839078:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432924672:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099387.839112:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432924672:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (104us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099387.839121:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1639
00000100:00100000:1.0:1766099387.839155:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432924672:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099387.839265:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924864
00000100:00100000:0.0:1766099387.839282:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2415
00000100:00100000:0.0:1766099387.839285:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b63cd4b7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432924736:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099387.839323:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b63cd4b7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432924736:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (612us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099387.839328:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2415
00000100:00100000:0.0:1766099387.839339:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2416
00000100:00100000:0.0:1766099387.839341:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c025e697 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432924864:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099387.839359:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c025e697 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432924864:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (592us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099387.839366:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2416
00000100:00100000:0.0:1766099387.839389:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de717044 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432924864:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099387.839396:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432924736:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099387.839503:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924800
00000100:00100000:3.0:1766099387.839527:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924928
00000100:00100000:3.0:1766099387.839549:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1083
00000100:00100000:3.0:1766099387.839553:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000199cfba3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432924928:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099387.839581:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000199cfba3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432924928:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (608us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099387.839585:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1083
00000100:00100000:3.0:1766099387.839594:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1492
00000100:00100000:3.0:1766099387.839597:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a38c4873 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432924800:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099387.839686:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099387.839711:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a38c4873 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432924800:12345-0@lo:8:kworker/u8:2.0 Request processed in 113us (896us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099387.839716:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1492
00010000:00080000:1.0:1766099387.839966:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001d84c111 x1851885432924928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099394 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099387.839975:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432924928:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099387.840025:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099387.840028:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099387.840031:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bb18fa0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432924800:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099388.349755:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099389.373708:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099390.397863:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099391.423315:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099392.286638:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099392.286683:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099392.286926:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099392.287105:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216768
00000100:00100000:2.0:1766099392.287151:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1640
00000100:00100000:2.0:1766099392.287157:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216768:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099392.287218:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216768:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 61us (535us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099392.287225:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1640
00000100:00100000:0.0:1766099392.287512:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216896
00000100:00100000:3.0:1766099392.288736:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216832
00000100:00100000:3.0:1766099392.288789:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2417
00000100:00100000:3.0:1766099392.288794:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000022d6139c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417216832:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099392.288850:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000022d6139c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417216832:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 56us (1925us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099392.288858:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2417
00000100:00100000:0.0:1766099392.288951:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1493
00000100:00100000:0.0:1766099392.288956:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbeb8a65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417216896:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099392.289057:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099392.289087:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbeb8a65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417216896:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 130us (2455us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099392.289093:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1493
00010000:00010000:1.0:1766099392.450082:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099392.958704:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099392.958712:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099392.958716:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099392.958723:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3896
00000100:00080000:1.0:1766099392.958727:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099392.958732:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099392.958752:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000de913b58 x1851885432925120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099392.958801:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000801bfca3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432925056:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099392.958804:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432925184:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099392.958872:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099392.958920:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432925120:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099392.958951:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099392.958993:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925120
00000100:00100000:1.0:1766099392.959030:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1494
00000100:00100000:1.0:1766099392.959036:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432925120:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099392.959135:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099392.959212:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925184
00000100:00100000:2.0:1766099392.959242:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2418
00000100:00100000:2.0:1766099392.959245:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006301b41a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432925184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099392.959362:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432924992:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099392.960079:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006301b41a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432925184:12345-0@lo:400:kworker/u8:3.0 Request processed in 831us (1209us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099392.960089:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2418
00000100:00100000:2.0:1766099392.960179:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432925248:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099392.960206:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099392.960235:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925248
00000100:00100000:2.0:1766099392.960272:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1084
00000100:00100000:2.0:1766099392.960276:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee68381b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432925248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099392.960316:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee68381b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432925248:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (109us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099392.960322:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1084
00010000:00080000:2.0:1766099392.960821:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003b84bdab x1851885432925248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099399 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099392.960834:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432925248:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099392.961000:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099392.961037:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432924992
00000100:00100000:3.0:1766099392.961060:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1641
00000100:00100000:3.0:1766099392.961063:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432924992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099392.961091:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432924992:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (96us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099392.961100:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1641
00000100:00100000:3.0:1766099392.961127:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432924992:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099392.961180:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432925184:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099392.961413:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432925120:12345-0@lo:8:kworker/u8:3.0 Request processed in 2375us (2461us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099392.961422:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099392.961425:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1494
00000100:00100000:1.0:1766099392.961463:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925056
00000100:00100000:1.0:1766099392.961520:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2419
00000100:00100000:1.0:1766099392.961525:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f68a282c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432925056:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099392.961546:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099392.961551:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099392.961557:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432925120:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099392.961565:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f68a282c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432925056:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (144us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099392.961572:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2419
00000100:00100000:0.0:1766099392.961635:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000801bfca3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432925056:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099393.470719:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099394.494775:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099395.517893:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099396.542699:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099397.407238:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099397.407294:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099397.407620:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417216960
00000100:00100000:2.0:1766099397.415515:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099397.415591:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217088
00000100:00100000:2.0:1766099397.415642:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1495
00000100:00100000:2.0:1766099397.415649:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092527c0a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417217088:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099397.415760:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099397.415761:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1642
00000100:00100000:3.0:1766099397.415770:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216960:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099397.415851:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092527c0a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417217088:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 201us (346us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099397.415863:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1495
00000100:00100000:0.0:1766099397.415880:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217024
00000100:00100000:0.0:1766099397.415919:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2420
00000100:00100000:0.0:1766099397.415924:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec1ae801 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417217024:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099397.417887:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417216960:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 2107us (10661us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099397.417906:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1642
00000100:00100000:0.0:1766099397.418189:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec1ae801 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417217024:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 2262us (10893us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099397.418200:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2420
00010000:00010000:0.0:1766099397.565745:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099398.077716:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099398.077724:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099398.077729:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099398.077739:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3901
00000100:00080000:1.0:1766099398.077743:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099398.077749:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099398.077773:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007382a6ac x1851885432925440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099398.077895:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432925376:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099398.077978:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099398.078027:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099398.078032:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000848a5086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432925504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099398.078055:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099398.078085:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432925440:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099398.078113:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099398.078202:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925376
00000100:00100000:1.0:1766099398.078275:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925504
00000100:00100000:1.0:1766099398.078301:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2421
00000100:00100000:1.0:1766099398.078308:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432925376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099398.078363:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432925376:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (386us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099398.078371:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2421
00000100:00100000:1.0:1766099398.078390:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2422
00000100:00100000:1.0:1766099398.078393:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432925504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099398.078416:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432925504:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (360us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099398.078421:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2422
00000100:00100000:1.0:1766099398.078468:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000848a5086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432925504:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099398.078509:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432925376:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099398.078680:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432925312:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099398.078720:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099398.078750:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099398.078755:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432925568:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099398.078777:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099398.078861:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925440
00000100:00100000:1.0:1766099398.078893:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925568
00000100:00100000:1.0:1766099398.078914:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1085
00000100:00100000:1.0:1766099398.078918:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037d37a0f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432925568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099398.078950:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037d37a0f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432925568:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (172us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099398.078957:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1085
00000100:00100000:1.0:1766099398.078969:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1496
00000100:00100000:1.0:1766099398.078972:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432925440:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099398.079079:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099398.079109:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432925440:12345-0@lo:8:kworker/u8:3.0 Request processed in 136us (996us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099398.079116:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1496
00000100:00100000:3.0:1766099398.079183:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925312
00000100:00100000:3.0:1766099398.079228:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1643
00000100:00100000:3.0:1766099398.079232:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432925312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099398.079269:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432925312:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (548us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099398.079276:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1643
00000100:00080000:3.0:1766099398.079318:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099398.079322:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099398.079327:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432925440:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766099398.079504:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000020b6667c x1851885432925568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099405 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099398.079516:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432925568:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099398.079526:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432925312:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099398.589812:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099399.613862:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099400.642009:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099401.663452:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099402.525452:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099402.525537:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099402.525564:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099402.525646:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217152
00000100:00100000:3.0:1766099402.525703:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1644
00000100:00100000:3.0:1766099402.525709:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417217152:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099402.525778:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417217152:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (334us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099402.525786:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1644
00000100:00100000:0.0:1766099402.526212:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217216
00000100:00100000:0.0:1766099402.526254:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2423
00000100:00100000:0.0:1766099402.526264:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417217216:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099402.526309:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417217216:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 44us (773us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099402.526315:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2423
00000100:00100000:2.0:1766099402.529196:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217280
00000100:00100000:2.0:1766099402.529245:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1497
00000100:00100000:2.0:1766099402.529251:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417217280:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099402.529348:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099402.529402:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417217280:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 151us (3839us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099402.529410:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1497
00010000:00010000:0.0:1766099402.686272:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099403.197677:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099403.197684:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099403.197687:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099403.197694:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3906
00000100:00080000:1.0:1766099403.197698:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099403.197703:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099403.197722:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000020b6667c x1851885432925760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099403.197955:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432925824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099403.198022:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099403.198053:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099403.198058:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432925696:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099403.198081:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099403.198088:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000801bfca3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432925888:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099403.198109:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432925760:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099403.198126:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099403.198144:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099403.198146:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099403.198153:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004bd75cc9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432925632:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099403.198216:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099403.198251:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925888
00000100:00100000:3.0:1766099403.198286:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925760
00000100:00100000:3.0:1766099403.198301:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1086
00000100:00100000:3.0:1766099403.198307:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c315443c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432925888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099403.198340:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1498
00000100:00100000:2.0:1766099403.198345:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432925760:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099403.198359:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c315443c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432925888:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (234us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099403.198367:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1086
00010000:00080000:3.0:1766099403.198409:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000801bfca3 x1851885432925888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099410 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099403.198420:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000801bfca3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432925888:0@lo:400:kworker/u8:3.0
00010000:02020000:2.0:1766099403.198445:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00080000:3.0:1766099403.199923:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099403.199930:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099403.199935:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432925760:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099403.199989:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432925760:12345-0@lo:8:kworker/u8:3.0 Request processed in 1642us (1845us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099403.199997:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1498
00000100:00100000:1.0:1766099403.200327:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925824
00000100:00100000:1.0:1766099403.200377:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925696
00000100:00100000:1.0:1766099403.200409:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2424
00000100:00100000:1.0:1766099403.200414:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432925824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099403.200472:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432925824:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (2451us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099403.200499:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2424
00000100:00100000:1.0:1766099403.200514:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2425
00000100:00100000:1.0:1766099403.200517:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432925696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099403.200540:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432925696:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (2461us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099403.200545:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2425
00000100:00100000:1.0:1766099403.200565:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925632
00000100:00100000:1.0:1766099403.200587:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1645
00000100:00100000:1.0:1766099403.200591:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432925632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099403.200624:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432925632:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (2409us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099403.200630:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1645
00000100:00100000:1.0:1766099403.200666:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432925696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099403.200676:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432925824:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099403.200722:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004bd75cc9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432925632:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099403.709729:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099404.737291:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099405.758153:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099406.781853:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099407.645654:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099407.645708:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099407.645813:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217344
00000100:00100000:2.0:1766099407.645869:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1646
00000100:00100000:2.0:1766099407.645876:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417217344:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099407.645942:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417217344:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 66us (234us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099407.645950:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1646
00000100:00100000:3.0:1766099407.646139:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217408
00000100:00100000:3.0:1766099407.646210:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2426
00000100:00100000:3.0:1766099407.646214:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004bd75cc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417217408:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099407.646261:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004bd75cc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417217408:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (614us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099407.646268:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2426
00000100:00100000:1.0:1766099407.646874:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099407.646946:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217472
00000100:00100000:0.0:1766099407.646977:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1499
00000100:00100000:0.0:1766099407.646982:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417217472:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099407.647069:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099407.647112:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417217472:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 129us (240us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099407.647119:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1499
00010000:00010000:1.0:1766099407.806011:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099408.318203:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099408.318211:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099408.318214:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099408.318223:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3911
00000100:00080000:1.0:1766099408.318227:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099408.318233:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099408.318253:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bbe45651 x1851885432926080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099408.318308:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432926208:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.318377:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099408.318409:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099408.318417:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432925952:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.318449:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099408.318503:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.318532:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099408.318555:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099408.318562:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432926144:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.318585:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099408.318633:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926208
00000100:00100000:1.0:1766099408.318679:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1087
00000100:00100000:1.0:1766099408.318686:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432926208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.318739:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432926208:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (364us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099408.318748:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1087
00000100:00100000:1.0:1766099408.318781:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432925952
00000100:00100000:1.0:1766099408.318813:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1647
00000100:00100000:1.0:1766099408.318819:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432925952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.318851:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432925952:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (403us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099408.318859:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1647
00000100:00100000:1.0:1766099408.318897:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926016
00000100:00100000:1.0:1766099408.318930:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926144
00000100:00100000:1.0:1766099408.318955:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2427
00000100:00100000:1.0:1766099408.318960:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bfca3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.319000:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bfca3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926016:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (469us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099408.319008:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2427
00000100:00100000:1.0:1766099408.319028:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2428
00000100:00100000:1.0:1766099408.319034:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb18fa0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432926144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.319064:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb18fa0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432926144:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (480us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099408.319071:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2428
00000100:00100000:1.0:1766099408.319149:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432926144:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.319168:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099408.319215:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432925952:0@lo:400:kworker/u8:2.0
00010000:00080000:1.0:1766099408.319232:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003b84bdab x1851885432926208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099415 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099408.319242:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432926208:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099408.320885:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432926080:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099408.320943:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099408.322191:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926080
00000100:00100000:0.0:1766099408.322241:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1500
00000100:00100000:0.0:1766099408.322247:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432926080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099408.322343:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099408.322412:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432926080:12345-0@lo:8:kworker/u8:2.0 Request processed in 164us (1468us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099408.322420:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1500
00000100:00080000:3.0:1766099408.322562:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099408.322567:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099408.322572:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432926080:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099408.829725:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099409.854610:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099410.877750:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099411.903388:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099412.770854:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099412.770904:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099412.770931:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217664
00000100:00100000:1.0:1766099412.770943:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099412.770980:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1501
00000100:00100000:2.0:1766099412.770985:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d84c111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417217664:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:3.0:1766099412.771087:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217536
00010000:02020000:2.0:1766099412.771095:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099412.771128:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1648
00000100:00100000:3.0:1766099412.771132:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417217536:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099412.771159:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417217536:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 27us (256us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099412.771160:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d84c111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417217664:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 173us (313us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099412.771167:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1501
00000100:00100000:3.0:1766099412.771168:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1648
00000100:00100000:0.0:1766099412.775366:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217600
00000100:00100000:0.0:1766099412.775432:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2429
00000100:00100000:0.0:1766099412.775439:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417217600:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099412.775532:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417217600:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 92us (4589us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099412.775540:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2429
00010000:00010000:0.0:1766099412.928437:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099413.438031:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099413.438040:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099413.438043:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099413.438052:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3916
00000100:00080000:3.0:1766099413.438056:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099413.438062:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099413.438085:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000055c81506 x1851885432926400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099413.438150:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2cdda05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926336:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099413.438214:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099413.438244:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099413.438250:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d94121f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432926464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099413.438271:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099413.438323:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432926400:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099413.438355:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099413.438395:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926336
00000100:00100000:3.0:1766099413.438438:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926464
00000100:00100000:3.0:1766099413.438465:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2430
00000100:00100000:3.0:1766099413.438470:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099413.438512:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2431
00000100:00100000:0.0:1766099413.438519:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432926464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099413.438552:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926336:12345-0@lo:400:kworker/u8:3.0 Request processed in 79us (337us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099413.438561:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2430
00000100:00100000:0.0:1766099413.438567:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432926464:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (293us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099413.438576:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2431
00000100:00100000:3.0:1766099413.438609:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d94121f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432926464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099413.438623:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2cdda05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926336:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099413.438855:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432926528:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099413.438886:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099413.438918:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099413.438924:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432926272:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099413.438948:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099413.439147:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926400
00000100:00100000:2.0:1766099413.439171:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926528
00000100:00100000:2.0:1766099413.439192:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088
00000100:00100000:2.0:1766099413.439196:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432926528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099413.439235:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432926528:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (349us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099413.439241:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088
00000100:00100000:2.0:1766099413.439253:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1502
00000100:00100000:2.0:1766099413.439256:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086a458df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432926400:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099413.439366:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:3.0:1766099413.439395:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006ca3c93d x1851885432926528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099420 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099413.439406:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432926528:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099413.439599:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086a458df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432926400:12345-0@lo:8:kworker/u8:3.0 Request processed in 340us (1244us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099413.439605:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1502
00000100:00080000:3.0:1766099413.439671:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099413.439675:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099413.439679:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432926400:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099413.442335:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926272
00000100:00100000:0.0:1766099413.442425:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1649
00000100:00100000:0.0:1766099413.442434:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432926272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099413.442591:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432926272:12345-0@lo:400:kworker/u8:3.0 Request processed in 157us (3641us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099413.442599:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1649
00000100:00100000:2.0:1766099413.442669:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432926272:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099413.957518:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099414.973715:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099415.997707:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099417.022065:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099417.885031:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099417.885084:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099417.885125:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099417.885145:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217728
00000100:00100000:0.0:1766099417.885206:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217792
00000100:00100000:1.0:1766099417.885210:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1650
00000100:00100000:1.0:1766099417.885218:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417217728:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099417.885243:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2432
00000100:00100000:0.0:1766099417.885248:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023d3ec95 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417217792:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099417.885290:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417217728:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 70us (204us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099417.885291:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023d3ec95 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417217792:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 39us (265us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099417.885305:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1650
00000100:00100000:0.0:1766099417.885310:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2432
00000100:00100000:3.0:1766099417.885553:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217856
00000100:00100000:3.0:1766099417.885588:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1503
00000100:00100000:3.0:1766099417.885596:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000179aab07 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417217856:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099417.885689:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099417.885728:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000179aab07 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417217856:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 135us (603us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099417.885735:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1503
00010000:00010000:0.0:1766099418.046044:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099418.557724:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099418.557734:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099418.557739:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099418.557749:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3921
00000100:00080000:2.0:1766099418.557754:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099418.557762:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099418.557785:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000083021c1c x1851885432926720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099418.557835:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007acd880d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432926848:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099418.557879:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432926592:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099418.557907:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099418.558597:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003e6f9e94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926656:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099418.558925:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099418.558982:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926592
00000100:00100000:1.0:1766099418.559017:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1651
00000100:00100000:1.0:1766099418.559022:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432926592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099418.559052:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083021c1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432926720:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099418.559724:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecfa9a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432926784:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099418.559965:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099418.559970:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099418.559974:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432926592:12345-0@lo:400:kworker/u8:2.0 Request processed in 947us (1055us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099418.559985:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1651
00000100:00100000:3.0:1766099418.560019:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926848
00000100:00100000:1.0:1766099418.560039:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432926592:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099418.560085:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926656
00000100:00100000:1.0:1766099418.560113:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2433
00000100:00100000:1.0:1766099418.560116:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926656:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099418.560135:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926720
00000100:00100000:1.0:1766099418.560167:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926656:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (204us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099418.560173:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2433
00000100:00100000:0.0:1766099418.560206:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099418.560209:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1504
00000100:00100000:1.0:1766099418.560214:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f745127a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432926720:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099418.560247:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926784
00000100:00100000:0.0:1766099418.560276:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2434
00000100:00100000:0.0:1766099418.560280:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432926784:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099418.560306:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099418.560313:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee5b88f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432926784:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (108us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099418.560321:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2434
00000100:00100000:1.0:1766099418.560351:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f745127a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432926720:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (384us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099418.560355:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecfa9a69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432926784:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099418.560367:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1504
00000100:00100000:3.0:1766099418.562159:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1089
00000100:00100000:3.0:1766099418.562168:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d96f6de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432926848:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099418.562252:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d96f6de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432926848:12345-0@lo:400:kworker/u8:2.0 Request processed in 84us (4351us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099418.562259:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1089
00000100:00100000:3.0:1766099418.562305:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003e6f9e94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926656:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099418.562354:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099418.562358:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099418.562362:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083021c1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432926720:0@lo:8:kworker/u8:2.0
00010000:00080000:0.0:1766099418.563771:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007acd880d x1851885432926848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099425 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099418.563786:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007acd880d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432926848:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099419.069695:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099420.095375:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099421.118918:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099422.145861:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099423.005352:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099423.005422:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099423.005470:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099423.005605:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218048
00000100:00100000:1.0:1766099423.005669:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1505
00000100:00100000:1.0:1766099423.005678:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd789e54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417218048:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099423.005791:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099423.005872:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd789e54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417218048:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 194us (532us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099423.005880:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1505
00000100:00100000:3.0:1766099423.006212:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217920
00000100:00100000:3.0:1766099423.006255:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1652
00000100:00100000:3.0:1766099423.006261:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417217920:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099423.006312:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417217920:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 50us (890us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099423.006320:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1652
00000100:00100000:0.0:1766099423.006397:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417217984
00000100:00100000:2.0:1766099423.006519:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2435
00000100:00100000:2.0:1766099423.006529:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061109c96 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417217984:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099423.006573:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061109c96 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417217984:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (1104us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099423.006580:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2435
00010000:00010000:2.0:1766099423.173570:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099423.677935:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099423.677945:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099423.677950:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099423.677959:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3927
00000100:00080000:2.0:1766099423.677963:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099423.677970:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099423.677997:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000dbe9c3c7 x1851885432927040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099423.678175:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c023a180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926976:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099423.678265:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099423.678322:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099423.678330:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8301247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432927104:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099423.678356:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099423.678394:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbe9c3c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432927040:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099423.678429:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099423.678517:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926976
00000100:00100000:3.0:1766099423.678641:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2436
00000100:00100000:3.0:1766099423.678649:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099423.678721:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432926976:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (457us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099423.678798:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c0ab926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432926912:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099423.678841:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099423.678863:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099423.678870:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085cf7081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432927168:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099423.678889:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099423.678913:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432926912
00000100:00100000:1.0:1766099423.678941:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1653
00000100:00100000:1.0:1766099423.678945:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432926912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099423.678972:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432926912:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (131us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099423.678979:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1653
00000100:00100000:1.0:1766099423.679015:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927040
00000100:00100000:1.0:1766099423.679043:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927168
00000100:00100000:1.0:1766099423.679065:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1090
00000100:00100000:1.0:1766099423.679069:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432927168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099423.679127:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432927168:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (237us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099423.679133:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1090
00000100:00100000:1.0:1766099423.679148:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1506
00000100:00100000:1.0:1766099423.679151:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432927040:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099423.679185:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2436
00000100:00100000:3.0:1766099423.679197:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927104
00000100:00100000:3.0:1766099423.679219:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2437
00000100:00100000:3.0:1766099423.679223:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2cdda05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432927104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099423.679709:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2cdda05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432927104:12345-0@lo:400:kworker/u8:2.0 Request processed in 483us (1351us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099423.679971:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2437
00000100:00100000:2.0:1766099423.680298:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8301247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432927104:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099423.680315:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c023a180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432926976:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766099423.680389:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000085cf7081 x1851885432927168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099430 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099423.680402:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085cf7081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432927168:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099423.680413:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c0ab926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432926912:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099423.682253:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099423.682374:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432927040:12345-0@lo:8:kworker/u8:2.0 Request processed in 3218us (3942us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099423.682389:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1506
00000100:00080000:3.0:1766099423.682525:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099423.682530:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099423.682536:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbe9c3c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432927040:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099424.189789:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099425.213773:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099426.238290:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099427.264161:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099428.124850:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099428.124917:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099428.124956:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099428.125041:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218112
00000100:00100000:2.0:1766099428.125125:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1654
00000100:00100000:2.0:1766099428.125132:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417218112:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099428.125198:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218176
00000100:00100000:2.0:1766099428.125217:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417218112:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 84us (300us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099428.125225:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1654
00000100:00100000:0.0:1766099428.129671:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2438
00000100:00100000:0.0:1766099428.129684:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a3dc47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417218176:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099428.129794:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a3dc47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417218176:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 110us (4955us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099428.129805:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2438
00000100:00100000:3.0:1766099428.130284:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218240
00000100:00100000:3.0:1766099428.130328:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1507
00000100:00100000:3.0:1766099428.130334:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024346c91 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417218240:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099428.130437:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099428.130514:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024346c91 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417218240:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 179us (5557us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099428.130523:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1507
00010000:00010000:0.0:1766099428.286821:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099428.801047:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099428.801055:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099428.801059:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099428.801066:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3932
00000100:00080000:3.0:1766099428.801070:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099428.801076:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099428.801101:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000be292ac7 x1851885432927360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099428.801157:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be292ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432927360:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801225:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099428.801282:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432927296:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801310:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099428.801328:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099428.801334:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cad6b0f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432927424:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801354:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099428.801390:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432927488:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801413:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099428.801429:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099428.801434:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432927232:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801459:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099428.801514:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927232
00000100:00100000:3.0:1766099428.801520:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927360
00000100:00100000:3.0:1766099428.801554:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927488
00000100:00100000:1.0:1766099428.801561:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1655
00000100:00100000:1.0:1766099428.801569:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432927232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801582:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091
00000100:00100000:3.0:1766099428.801587:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432927488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099428.801625:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432927232:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (164us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099428.801632:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432927488:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (217us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099428.801635:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1655
00000100:00100000:3.0:1766099428.801645:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091
00000100:00100000:3.0:1766099428.801662:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1508
00000100:00100000:3.0:1766099428.801666:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfede6c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432927360:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099428.801676:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927296
00000100:00100000:1.0:1766099428.801700:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927424
00000100:00100000:1.0:1766099428.801715:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2439
00000100:00100000:1.0:1766099428.801719:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000032cf85a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432927296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099428.801754:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000032cf85a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432927296:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (443us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099428.801760:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2439
00010000:02020000:3.0:1766099428.801770:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099428.801774:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2440
00000100:00100000:1.0:1766099428.801779:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004aef0954 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432927424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801813:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfede6c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432927360:12345-0@lo:8:kworker/u8:3.0 Request processed in 144us (587us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099428.801819:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004aef0954 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432927424:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (464us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099428.801823:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1508
00000100:00100000:1.0:1766099428.801832:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2440
00000100:00100000:3.0:1766099428.801878:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cad6b0f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432927424:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099428.801890:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432927232:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099428.801895:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432927296:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766099428.801910:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c9d7b580 x1851885432927488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099435 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099428.801919:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9d7b580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432927488:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099428.801939:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099428.801944:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099428.801948:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be292ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432927360:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099429.309959:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099430.333661:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099431.361155:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099432.384187:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099433.245138:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099433.245262:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218304
00000100:00100000:2.0:1766099433.245366:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1656
00000100:00100000:2.0:1766099433.245375:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417218304:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099433.245595:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417218304:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 219us (468us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099433.245605:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1656
00000100:00100000:3.0:1766099433.248279:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099433.250133:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099433.250837:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218368
00000100:00100000:0.0:1766099433.251878:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2441
00000100:00100000:0.0:1766099433.251890:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000220e4534 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417218368:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099433.252023:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218432
00000100:00100000:1.0:1766099433.252309:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1509
00000100:00100000:1.0:1766099433.252315:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417218432:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099433.252421:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099433.252531:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417218432:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 215us (2406us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099433.252539:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1509
00000100:00100000:0.0:1766099433.252822:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000220e4534 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417218368:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 932us (4553us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099433.252832:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2441
00010000:00010000:0.0:1766099433.406373:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099433.917860:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099433.917868:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099433.917873:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099433.917883:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3937
00000100:00080000:2.0:1766099433.917888:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099433.917896:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099433.917921:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c023a180 x1851885432927680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099433.918018:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8301247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432927744:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918108:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099433.918155:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099433.918160:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085cf7081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432927616:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918184:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099433.918219:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c023a180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432927680:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918248:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099433.918274:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927744
00000100:00100000:3.0:1766099433.918286:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927680
00000100:00100000:3.0:1766099433.918328:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1510
00000100:00100000:3.0:1766099433.918335:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432927680:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099433.918342:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927616
00000100:00100000:0.0:1766099433.918363:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2442
00000100:00100000:0.0:1766099433.918367:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be292ac7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432927744:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:3.0:1766099433.918433:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099433.918447:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be292ac7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432927744:12345-0@lo:400:kworker/u8:2.0 Request processed in 77us (339us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099433.918455:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2442
00000100:00100000:3.0:1766099433.918499:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432927680:12345-0@lo:8:kworker/u8:2.0 Request processed in 141us (229us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099433.918508:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1510
00000100:00100000:0.0:1766099433.918514:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2443
00000100:00100000:0.0:1766099433.918519:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432927616:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918536:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c0ab926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432927552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099433.918562:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432927616:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (377us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099433.918569:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2443
00000100:00100000:3.0:1766099433.918579:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099433.918604:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099433.918609:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080c118b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432927808:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918629:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:3.0:1766099433.918677:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099433.918682:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099433.918687:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c023a180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432927680:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099433.918701:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927808
00000100:00100000:3.0:1766099433.918718:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927552
00000100:00100000:0.0:1766099433.918741:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092
00000100:00100000:0.0:1766099433.918750:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432927808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918753:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1657
00000100:00100000:3.0:1766099433.918760:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432927552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099433.918807:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cad6b0f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432927808:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (176us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099433.918811:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432927552:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (231us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099433.918823:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1657
00000100:00100000:0.0:1766099433.918828:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092
00010000:00080000:3.0:1766099433.918869:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000080c118b3 x1851885432927808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099440 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099433.918880:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080c118b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432927808:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099433.918891:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085cf7081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432927616:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099433.918894:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c0ab926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432927552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099433.918917:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8301247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432927744:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099434.431225:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099435.458099:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099436.479072:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099437.501879:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099438.367142:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099438.369237:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218496
00000100:00100000:2.0:1766099438.369321:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1658
00000100:00100000:2.0:1766099438.369331:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417218496:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099438.369437:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417218496:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 106us (2307us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099438.369445:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1658
00000100:00100000:1.0:1766099438.369545:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099438.370117:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099438.370525:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218560
00000100:00100000:1.0:1766099438.370543:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218624
00000100:00100000:2.0:1766099438.370578:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2444
00000100:00100000:2.0:1766099438.370584:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417218560:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099438.370618:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1511
00000100:00100000:1.0:1766099438.370623:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417218624:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:2.0:1766099438.370652:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417218560:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (1516us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099438.370660:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2444
00010000:02020000:1.0:1766099438.370794:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099438.370841:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417218624:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 217us (725us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099438.370848:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1511
00010000:00010000:2.0:1766099438.527635:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099439.040076:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099439.040086:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099439.040090:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099439.040112:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3942
00000100:00080000:3.0:1766099439.040116:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099439.040125:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099439.040163:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000039dc5372 x1851885432928000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099439.040225:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432928128:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099439.040232:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432927872:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099439.040430:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432928064:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099439.041202:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432927936:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099439.041282:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099439.041340:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928000:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099439.041368:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099439.041428:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928128
00000100:00100000:3.0:1766099439.041500:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928000
00000100:00100000:3.0:1766099439.041704:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1093
00000100:00100000:3.0:1766099439.041713:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002df41de4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432928128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099439.041779:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002df41de4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432928128:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (499us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099439.041788:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1093
00000100:00100000:3.0:1766099439.041810:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1512
00000100:00100000:3.0:1766099439.041814:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad1e01cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432928000:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099439.041934:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099439.041970:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad1e01cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432928000:12345-0@lo:8:kworker/u8:2.0 Request processed in 154us (601us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099439.041977:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1512
00000100:00080000:3.0:1766099439.042046:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099439.042050:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099439.042055:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928000:0@lo:8:kworker/u8:2.0
00010000:00080000:3.0:1766099439.042090:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000f198af5 x1851885432928128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099446 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099439.042101:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432928128:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099439.043650:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099439.043651:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099439.043861:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099439.044143:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928064
00000100:00100000:1.0:1766099439.044200:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927936
00000100:00100000:1.0:1766099439.044226:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2445
00000100:00100000:1.0:1766099439.044232:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6a67406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432928064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099439.044290:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6a67406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432928064:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (648us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099439.044298:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2445
00000100:00100000:1.0:1766099439.044315:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2446
00000100:00100000:1.0:1766099439.044318:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7e31e7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432927936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099439.044349:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7e31e7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432927936:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (489us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099439.044355:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2446
00000100:00100000:1.0:1766099439.044393:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432927936:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099439.044424:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432928064:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099439.046732:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432927872
00000100:00100000:3.0:1766099439.046965:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1659
00000100:00100000:3.0:1766099439.046975:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432927872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099439.047094:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432927872:12345-0@lo:400:kworker/u8:2.0 Request processed in 120us (3453us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099439.047369:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432927872:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099439.048049:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1659
00010000:00010000:0.0:1766099439.549761:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099440.578467:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099441.598770:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099442.621877:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099443.485257:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099443.487489:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099443.487915:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218688
00000100:00100000:2.0:1766099443.488834:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1660
00000100:00100000:2.0:1766099443.488846:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417218688:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099443.489152:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417218688:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 307us (3907us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099443.489162:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1660
00000100:00100000:0.0:1766099443.490301:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218752
00000100:00100000:0.0:1766099443.490367:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2447
00000100:00100000:0.0:1766099443.490374:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417218752:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099443.490440:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417218752:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 66us (2974us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099443.490449:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2447
00000100:00100000:2.0:1766099443.490928:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099443.492191:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218816
00000100:00100000:1.0:1766099443.492261:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1513
00000100:00100000:1.0:1766099443.492268:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8301247 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417218816:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099443.492373:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099443.492463:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8301247 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417218816:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 194us (1537us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099443.492502:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1513
00010000:00010000:0.0:1766099443.646201:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099444.159454:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099444.159464:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099444.159469:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099444.159501:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3947
00000100:00080000:0.0:1766099444.159506:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099444.159513:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099444.159539:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007b5b6926 x1851885432928320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099444.159550:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432928256:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099444.159611:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432928384:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099444.159646:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099444.159723:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928256
00000100:00100000:1.0:1766099444.159779:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2448
00000100:00100000:1.0:1766099444.159787:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432928256:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099444.159844:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432928256:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (200us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099444.159853:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2448
00000100:00100000:1.0:1766099444.159893:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099444.159952:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432928256:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099444.159977:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928384
00000100:00100000:1.0:1766099444.159997:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2449
00000100:00100000:1.0:1766099444.160001:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432928384:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099444.160029:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432928384:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (137us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099444.160035:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2449
00000100:00100000:1.0:1766099444.160059:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432928384:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160227:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432928448:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160262:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099444.160293:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099444.160298:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d779487b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432928192:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160321:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099444.160343:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928320:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160363:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099444.160390:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928448
00000100:00100000:3.0:1766099444.160412:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928320
00000100:00100000:3.0:1766099444.160426:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1094
00000100:00100000:3.0:1766099444.160430:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f198af5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432928448:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160458:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f198af5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432928448:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (196us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099444.160464:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1094
00000100:00100000:3.0:1766099444.160474:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1514
00000100:00100000:3.0:1766099444.160498:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049dffb54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432928320:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099444.160593:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099444.160627:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049dffb54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432928320:12345-0@lo:8:kworker/u8:3.0 Request processed in 128us (263us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099444.160633:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1514
00000100:00080000:3.0:1766099444.160668:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099444.160672:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099444.160676:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928320:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766099444.160701:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005fa10d21 x1851885432928448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099451 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099444.160710:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fa10d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432928448:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160729:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928192
00000100:00100000:3.0:1766099444.160754:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1661
00000100:00100000:3.0:1766099444.160758:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432928192:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099444.160789:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432928192:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (467us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099444.160795:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1661
00000100:00100000:3.0:1766099444.160825:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d779487b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432928192:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099444.669849:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099445.694299:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099446.718449:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099447.741745:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099448.604588:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099448.604651:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099448.604776:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218880
00000100:00100000:0.0:1766099448.604778:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417218944
00000100:00100000:2.0:1766099448.604888:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1662
00000100:00100000:0.0:1766099448.604889:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2450
00000100:00100000:2.0:1766099448.604913:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417218880:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099448.604914:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417218944:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099448.605047:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417218880:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 145us (470us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099448.605058:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1662
00000100:00100000:0.0:1766099448.605389:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417218944:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 486us (735us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099448.605399:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2450
00000100:00100000:2.0:1766099448.617331:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099448.617537:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219008
00000100:00100000:1.0:1766099448.617610:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1515
00000100:00100000:1.0:1766099448.617617:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085cf7081 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417219008:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099448.617722:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099448.617801:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085cf7081 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417219008:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 183us (483us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099448.617809:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1515
00010000:00010000:2.0:1766099448.766215:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099449.277769:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099449.277777:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099449.277781:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099449.277790:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3952
00000100:00080000:1.0:1766099449.277794:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099449.277799:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099449.277811:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432928576:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.277825:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bf96017c x1851885432928640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099449.277901:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432928704:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099449.277942:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099449.278467:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928640:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099449.278651:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099449.278713:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928576
00000100:00100000:1.0:1766099449.278748:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928704
00000100:00100000:1.0:1766099449.278764:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2451
00000100:00100000:1.0:1766099449.278769:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432928576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.278813:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fa10d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432928576:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (876us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099449.278819:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2451
00000100:00100000:1.0:1766099449.278840:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2452
00000100:00100000:1.0:1766099449.278845:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432928704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.278869:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432928704:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (222us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099449.278874:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2452
00000100:00100000:1.0:1766099449.278908:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432928704:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.278954:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432928768:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.278977:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099449.278993:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099449.278997:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432928512:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.279015:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099449.279035:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928768
00000100:00100000:1.0:1766099449.279056:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095
00000100:00100000:1.0:1766099449.279059:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432928768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.279091:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432928768:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (113us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099449.279096:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095
00010000:00080000:1.0:1766099449.279122:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003b84bdab x1851885432928768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099456 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099449.279130:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432928768:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.279147:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928512
00000100:00100000:1.0:1766099449.279167:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1663
00000100:00100000:1.0:1766099449.279172:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432928512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099449.279200:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432928512:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (184us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099449.279205:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1663
00000100:00100000:1.0:1766099449.279233:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432928512:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099449.282351:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432928576:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099449.282775:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099449.283127:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928640
00000100:00100000:0.0:1766099449.283179:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1516
00000100:00100000:0.0:1766099449.283186:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd5b744c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432928640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099449.283284:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099449.283347:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd5b744c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432928640:12345-0@lo:8:kworker/u8:3.0 Request processed in 160us (574us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099449.283354:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1516
00000100:00080000:3.0:1766099449.283697:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099449.283703:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099449.283710:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928640:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099449.789700:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099450.813679:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099451.839921:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099452.861894:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099453.724358:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099453.724423:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099453.724543:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219136
00000100:00100000:1.0:1766099453.724617:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2453
00000100:00100000:1.0:1766099453.724625:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c0ab926 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417219136:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099453.724734:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c0ab926 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417219136:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 109us (311us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099453.724744:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2453
00000100:00100000:0.0:1766099453.725296:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219072
00000100:00100000:1.0:1766099453.726239:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1664
00000100:00100000:1.0:1766099453.726248:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417219072:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099453.726332:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417219072:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 85us (1987us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099453.726340:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1664
00000100:00100000:2.0:1766099453.726866:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099453.726946:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219200
00000100:00100000:3.0:1766099453.726984:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1517
00000100:00100000:3.0:1766099453.726988:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080c118b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417219200:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099453.727096:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099453.727147:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080c118b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417219200:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 157us (283us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099453.727155:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1517
00010000:00010000:2.0:1766099453.889643:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099454.398007:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099454.398017:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099454.398022:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099454.398032:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3957
00000100:00080000:3.0:1766099454.398036:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099454.398053:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099454.398088:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000f198af5 x1851885432928960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099454.398144:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928960:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099454.398232:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099454.398295:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928960
00000100:00100000:3.0:1766099454.398337:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1518
00000100:00100000:3.0:1766099454.398343:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432928960:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099454.398441:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099454.398519:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432928960:12345-0@lo:8:kworker/u8:2.0 Request processed in 174us (289us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099454.398526:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1518
00000100:00080000:3.0:1766099454.398571:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099454.398575:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099454.398578:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432928960:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398650:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432929088:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398679:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099454.398703:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099454.398707:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432928832:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398731:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099454.398751:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432929024:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398774:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099454.398787:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099454.398791:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432928896:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398819:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099454.398845:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929024
00000100:00100000:1.0:1766099454.398873:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928896
00000100:00100000:1.0:1766099454.398890:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2454
00000100:00100000:1.0:1766099454.398894:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de913b58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432929024:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398935:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de913b58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432929024:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (160us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099454.398945:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2454
00000100:00100000:1.0:1766099454.398956:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2455
00000100:00100000:1.0:1766099454.398959:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000413b47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432928896:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.398978:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000413b47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432928896:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (159us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099454.398983:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2455
00000100:00100000:1.0:1766099454.399014:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432928896:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099454.399022:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432929024:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099454.399148:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432928832
00000100:00100000:0.0:1766099454.399181:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1665
00000100:00100000:0.0:1766099454.399184:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432928832:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099454.399227:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432928832:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (496us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099454.399232:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1665
00000100:00100000:2.0:1766099454.399762:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432928832:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099454.400601:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929088
00000100:00100000:3.0:1766099454.400662:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1096
00000100:00100000:3.0:1766099454.400669:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b84bdab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432929088:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099454.400758:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b84bdab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432929088:12345-0@lo:400:kworker/u8:2.0 Request processed in 90us (2078us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099454.400765:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1096
00010000:00080000:2.0:1766099454.400869:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d7a3c8dd x1851885432929088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099461 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099454.400883:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432929088:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099454.916938:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099455.936188:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099456.960496:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099457.986206:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099458.846161:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099458.846226:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099458.846255:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219328
00000100:00100000:2.0:1766099458.846283:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099458.846326:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2456
00000100:00100000:0.0:1766099458.846334:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c023a180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417219328:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099458.846379:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219392
00000100:00100000:1.0:1766099458.846422:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1519
00000100:00100000:1.0:1766099458.846428:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbe9c3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417219392:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766099458.846435:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c023a180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417219328:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 99us (286us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099458.846444:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2456
00010000:02020000:1.0:1766099458.846569:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099458.846602:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbe9c3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417219392:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 174us (377us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099458.846610:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1519
00000100:00100000:3.0:1766099458.846848:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219264
00000100:00100000:3.0:1766099458.846888:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1666
00000100:00100000:3.0:1766099458.846893:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417219264:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099458.846940:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417219264:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (657us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099458.846947:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1666
00010000:00010000:0.0:1766099459.005791:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099459.517973:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099459.517982:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099459.517986:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099459.517996:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3962
00000100:00080000:3.0:1766099459.518000:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099459.518007:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099459.518029:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000049dffb54 x1851885432929280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099459.518132:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432929344:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099459.518320:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099459.518360:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099459.518365:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432929216:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099459.518385:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099459.518406:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432929280:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099459.518428:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099459.518464:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929280
00000100:00100000:3.0:1766099459.518520:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1520
00000100:00100000:3.0:1766099459.518526:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432929280:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099459.518620:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099459.518658:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432929280:12345-0@lo:8:kworker/u8:2.0 Request processed in 131us (229us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099459.518664:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1520
00000100:00080000:3.0:1766099459.518707:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099459.518710:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099459.518714:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049dffb54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432929280:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099459.518857:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929344
00000100:00100000:0.0:1766099459.518889:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929216
00000100:00100000:0.0:1766099459.518904:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2457
00000100:00100000:0.0:1766099459.518907:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007fb05d30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432929344:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099459.518951:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007fb05d30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432929344:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (633us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099459.518956:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2457
00000100:00100000:0.0:1766099459.518969:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2458
00000100:00100000:0.0:1766099459.518971:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000617de322 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432929216:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099459.518988:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000617de322 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432929216:12345-0@lo:400:kworker/u8:2.0 Request processed in 16us (603us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099459.518993:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2458
00000100:00100000:0.0:1766099459.519024:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002df41de4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432929216:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099459.519032:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad1e01cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432929344:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099459.519289:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432929152:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099459.519327:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099459.519355:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099459.519359:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432929408:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099459.519377:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099459.519471:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929152
00000100:00100000:0.0:1766099459.519517:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1667
00000100:00100000:0.0:1766099459.519521:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432929152:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099459.519557:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432929152:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (230us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099459.519563:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1667
00000100:00100000:2.0:1766099459.519648:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7a3c8dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432929152:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099459.519698:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929408
00000100:00100000:3.0:1766099459.519753:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1097
00000100:00100000:3.0:1766099459.519765:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007acd880d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432929408:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099459.519804:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007acd880d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432929408:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (426us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099459.519809:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1097
00010000:00080000:0.0:1766099459.520543:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000f198af5 x1851885432929408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099466 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099459.520557:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f198af5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432929408:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099460.030843:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099461.056912:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099462.080291:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099463.102034:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099463.969049:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099463.969109:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099463.969218:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099463.969345:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219456
00000100:00100000:1.0:1766099463.969418:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1668
00000100:00100000:1.0:1766099463.969426:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417219456:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099463.969555:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417219456:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 128us (518us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099463.969565:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1668
00000100:00100000:3.0:1766099463.971636:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219584
00000100:00100000:3.0:1766099463.971737:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1521
00000100:00100000:3.0:1766099463.971746:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e6f9e94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417219584:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099463.971916:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099463.972026:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e6f9e94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417219584:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 279us (2808us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099463.972035:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1521
00000100:00100000:0.0:1766099463.972134:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219520
00000100:00100000:0.0:1766099463.972184:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2459
00000100:00100000:0.0:1766099463.972190:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000083021c1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417219520:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099463.972259:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000083021c1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417219520:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (3150us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099463.972266:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2459
00010000:00010000:2.0:1766099464.125817:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099464.638413:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099464.638423:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099464.638428:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099464.638438:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3967
00000100:00080000:0.0:1766099464.638442:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099464.638449:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099464.638474:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000492edb9a x1851885432929600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099464.638546:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432929536:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099464.638624:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099464.638661:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099464.638666:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432929664:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099464.638689:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099464.638741:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929536
00000100:00100000:0.0:1766099464.638788:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929664
00000100:00100000:0.0:1766099464.638809:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2460
00000100:00100000:0.0:1766099464.638816:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfede6c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432929536:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099464.638868:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfede6c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432929536:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (245us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099464.638876:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2460
00000100:00100000:0.0:1766099464.638896:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2461
00000100:00100000:0.0:1766099464.638899:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432929664:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099464.638922:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9d7b580 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432929664:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (233us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099464.638927:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2461
00000100:00100000:0.0:1766099464.638974:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432929664:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099464.638984:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432929536:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099464.642137:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432929472:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099464.642239:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099464.642292:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099464.642299:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432929728:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099464.642329:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099464.642406:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929728
00000100:00100000:3.0:1766099464.642440:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432929600:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099464.642472:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1098
00000100:00100000:3.0:1766099464.642472:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099464.642507:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000199cfba3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432929728:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099464.642531:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929600
00000100:00100000:3.0:1766099464.642555:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1522
00000100:00100000:3.0:1766099464.642559:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007bef605c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432929600:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099464.642567:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000199cfba3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432929728:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (238us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099464.642573:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1098
00010000:02020000:3.0:1766099464.642675:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:1.0:1766099464.642688:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000adfcc04b x1851885432929728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099471 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099464.642703:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432929728:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099464.642734:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007bef605c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432929600:12345-0@lo:8:kworker/u8:2.0 Request processed in 174us (261us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099464.642741:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1522
00000100:00100000:2.0:1766099464.643510:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929472
00000100:00100000:2.0:1766099464.643555:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1669
00000100:00100000:2.0:1766099464.643559:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432929472:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099464.643614:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432929472:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (1378us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099464.643620:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1669
00000100:00100000:3.0:1766099464.644123:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432929472:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099464.644678:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099464.644683:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099464.644688:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432929600:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766099465.149792:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099466.174287:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099467.197755:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099468.224050:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099469.090530:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099469.090603:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099469.090611:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219712
00000100:00100000:1.0:1766099469.090649:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099469.090679:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2462
00000100:00100000:0.0:1766099469.090698:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbeb8a65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417219712:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099469.090811:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbeb8a65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417219712:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 123us (291us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099469.090820:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2462
00000100:00100000:3.0:1766099469.090828:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219776
00000100:00100000:3.0:1766099469.090854:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1523
00000100:00100000:3.0:1766099469.090859:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417219776:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099469.090957:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099469.090990:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417219776:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 131us (341us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099469.090997:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1523
00000100:00100000:2.0:1766099469.091075:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219648
00000100:00100000:2.0:1766099469.091114:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1670
00000100:00100000:2.0:1766099469.091119:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417219648:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099469.091148:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417219648:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 28us (546us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099469.091156:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1670
00010000:00010000:2.0:1766099469.245780:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099469.757734:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099469.757744:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099469.757748:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099469.757758:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3973
00000100:00080000:0.0:1766099469.757762:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099469.757768:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099469.757795:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001d84c111 x1851885432929920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099469.757851:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432929856:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.757936:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099469.757982:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099469.757988:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432929984:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.758010:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099469.758036:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432929920:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099469.758060:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099469.758105:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929856
00000100:00100000:1.0:1766099469.758134:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432930048:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.758156:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929984
00000100:00100000:1.0:1766099469.758176:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099469.758179:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2463
00000100:00100000:0.0:1766099469.758185:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432929856:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099469.758197:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099469.758202:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432929792:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099469.758363:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929920
00000100:00100000:2.0:1766099469.758387:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930048
00000100:00100000:2.0:1766099469.758409:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1099
00000100:00100000:2.0:1766099469.758414:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432930048:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.758524:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432929856:12345-0@lo:400:kworker/u8:2.0 Request processed in 339us (588us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099469.758532:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2463
00000100:00100000:0.0:1766099469.758555:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2464
00000100:00100000:0.0:1766099469.758558:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432929984:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.758584:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432929984:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (573us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099469.758590:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2464
00000100:00100000:0.0:1766099469.758634:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432929984:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.758646:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432929856:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099469.758694:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1524
00000100:00100000:3.0:1766099469.758699:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432929920:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099469.758789:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099469.758833:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432929920:12345-0@lo:8:kworker/u8:2.0 Request processed in 133us (772us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099469.758839:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1524
00000100:00080000:3.0:1766099469.758875:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099469.758879:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099469.758883:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432929920:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099469.761833:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099469.761909:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432929792
00000100:00100000:0.0:1766099469.761982:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1671
00000100:00100000:0.0:1766099469.761989:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432929792:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099469.762077:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432929792:12345-0@lo:400:kworker/u8:2.0 Request processed in 89us (255us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099469.762085:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1671
00000100:00100000:1.0:1766099469.762203:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432929792:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099469.762578:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432930048:12345-0@lo:400:kworker/u8:2.0 Request processed in 4161us (4399us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099469.762588:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1099
00010000:00080000:1.0:1766099469.762679:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000024346c91 x1851885432930048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099476 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099469.762704:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432930048:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099470.270339:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099471.294986:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099472.322357:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099473.342719:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099474.204990:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099474.205045:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099474.205143:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099474.205275:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219904
00000100:00100000:3.0:1766099474.205355:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2465
00000100:00100000:3.0:1766099474.205384:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417219904:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099474.205536:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417219904:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 171us (556us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099474.205546:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2465
00000100:00100000:2.0:1766099474.205736:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219968
00000100:00100000:2.0:1766099474.205772:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1525
00000100:00100000:2.0:1766099474.205777:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417219968:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099474.205879:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099474.205925:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417219968:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 147us (782us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099474.205932:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1525
00000100:00100000:0.0:1766099474.206121:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417219840
00000100:00100000:0.0:1766099474.206158:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1672
00000100:00100000:0.0:1766099474.206162:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417219840:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099474.206201:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417219840:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 38us (1156us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099474.206207:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1672
00010000:00010000:0.0:1766099474.365776:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099474.879226:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e82385d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432930112:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099474.879318:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099474.879469:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930112
00000100:00100000:0.0:1766099474.879547:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1673
00000100:00100000:0.0:1766099474.879554:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432930112:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099474.880430:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432930112:12345-0@lo:400:kworker/u8:2.0 Request processed in 871us (1109us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099474.880443:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1673
00000100:00100000:0.0:1766099474.880653:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d65d7291 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432930176:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099474.880697:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099474.880762:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930176
00000100:00100000:2.0:1766099474.880803:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2466
00000100:00100000:2.0:1766099474.880807:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086a458df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432930176:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099474.880928:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e82385d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432930112:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099474.881085:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099474.881092:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099474.881096:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099474.881101:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3978
00000100:00080000:3.0:1766099474.881103:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099474.881108:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099474.881136:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000a02de30 x1851885432930240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099474.881185:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a02de30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432930240:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099474.881218:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099474.881571:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086a458df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432930176:12345-0@lo:400:kworker/u8:2.0 Request processed in 761us (873us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099474.881579:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2466
00000100:00100000:2.0:1766099474.881614:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930240
00000100:00100000:2.0:1766099474.881637:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1526
00000100:00100000:2.0:1766099474.881640:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432930240:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099474.881753:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099474.881820:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d65d7291 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432930176:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099474.883984:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432930240:12345-0@lo:8:kworker/u8:2.0 Request processed in 2334us (2758us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099474.884002:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1526
00000100:00100000:2.0:1766099474.884131:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099474.884744:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432930304:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099474.884822:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099474.884888:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930304
00000100:00100000:2.0:1766099474.884941:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2467
00000100:00100000:2.0:1766099474.884947:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfa9a69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432930304:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099474.885001:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfa9a69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432930304:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (180us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099474.885008:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2467
00000100:00100000:2.0:1766099474.885057:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432930304:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099474.885447:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099474.885458:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9fe64a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432930368:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099474.885514:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766099474.885928:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099474.885934:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099474.885939:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a02de30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432930240:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099474.886849:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930368
00000100:00100000:2.0:1766099474.886918:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100
00000100:00100000:2.0:1766099474.886924:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432930368:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099474.887029:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432930368:12345-0@lo:400:kworker/u8:2.0 Request processed in 104us (1515us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099474.887036:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100
00010000:00080000:3.0:1766099474.893003:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f9fe64a8 x1851885432930368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099481 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099474.893031:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9fe64a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432930368:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099475.391313:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099476.414501:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099477.441408:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099478.463858:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099479.325428:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099479.325521:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099479.325569:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099479.325677:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220160
00000100:00100000:0.0:1766099479.325790:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1527
00000100:00100000:0.0:1766099479.325799:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9fe64a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417220160:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099479.326015:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099479.326138:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9fe64a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417220160:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 338us (720us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099479.326144:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220096
00000100:00100000:0.0:1766099479.326149:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1527
00000100:00100000:1.0:1766099479.326184:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2468
00000100:00100000:1.0:1766099479.326190:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e40221d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417220096:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099479.326234:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e40221d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417220096:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 44us (665us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099479.326240:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2468
00000100:00100000:2.0:1766099479.326335:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220032
00000100:00100000:2.0:1766099479.326374:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1674
00000100:00100000:2.0:1766099479.326379:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417220032:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099479.326407:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417220032:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 28us (888us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099479.326413:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1674
00010000:00010000:2.0:1766099479.485772:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099479.997856:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099479.997864:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099479.997868:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099479.997878:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3983
00000100:00080000:1.0:1766099479.997882:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099479.997890:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099479.997916:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a8d6226a x1851885432930560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099479.997999:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432930560:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099479.998081:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099479.998304:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432930624:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099479.998337:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099479.998356:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099479.998361:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432930496:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099479.998376:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099479.998408:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930624
00000100:00100000:3.0:1766099479.998702:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432930432:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099479.998755:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099479.998781:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099479.998786:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ed1b667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432930688:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099479.998810:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099479.998843:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930560
00000100:00100000:3.0:1766099479.998871:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930688
00000100:00100000:2.0:1766099479.998879:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1528
00000100:00100000:2.0:1766099479.998889:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1de4d22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432930560:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099479.998895:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101
00000100:00100000:3.0:1766099479.998901:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001415cdc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432930688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099479.998955:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001415cdc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432930688:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (144us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099479.998962:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101
00010000:02020000:2.0:1766099479.998986:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:3.0:1766099479.999013:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007ed1b667 x1851885432930688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099486 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099479.999028:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ed1b667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432930688:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099479.999039:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1de4d22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432930560:12345-0@lo:8:kworker/u8:3.0 Request processed in 150us (957us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099479.999048:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1528
00000100:00080000:3.0:1766099479.999076:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099479.999081:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099479.999085:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8d6226a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432930560:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099479.999170:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930496
00000100:00100000:0.0:1766099479.999196:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2469
00000100:00100000:0.0:1766099479.999202:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432930624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099479.999205:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2470
00000100:00100000:2.0:1766099479.999213:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432930496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099479.999251:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432930624:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (913us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099479.999259:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432930496:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (880us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099479.999260:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2469
00000100:00100000:2.0:1766099479.999269:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2470
00000100:00100000:0.0:1766099479.999294:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930432
00000100:00100000:0.0:1766099479.999317:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1675
00000100:00100000:0.0:1766099479.999321:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432930432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099479.999365:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432930432:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (611us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099479.999371:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1675
00000100:00100000:0.0:1766099479.999402:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432930496:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099479.999410:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432930624:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099480.000644:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adfcc04b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432930432:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099480.512110:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099481.533979:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099482.559192:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099483.581838:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099484.444516:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099484.444593:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099484.444718:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220224
00000100:00100000:1.0:1766099484.444778:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1676
00000100:00100000:1.0:1766099484.444791:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417220224:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099484.444885:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417220224:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 91us (286us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099484.444894:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1676
00000100:00100000:3.0:1766099484.445292:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099484.445371:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220352
00000100:00100000:2.0:1766099484.445411:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1529
00000100:00100000:2.0:1766099484.445418:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077d8956b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417220352:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099484.445544:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099484.445599:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077d8956b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417220352:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 180us (308us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099484.445608:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1529
00000100:00100000:0.0:1766099484.447896:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220288
00000100:00100000:0.0:1766099484.447983:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2471
00000100:00100000:0.0:1766099484.447990:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417220288:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099484.448099:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417220288:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 108us (3631us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099484.448107:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2471
00010000:00010000:2.0:1766099484.605764:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099485.119811:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432930752:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099485.119914:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099485.121913:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930752
00000100:00080000:0.0:1766099485.122157:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099485.122164:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099485.122168:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099485.122178:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3988
00000100:00080000:0.0:1766099485.122181:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099485.122188:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099485.122213:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000492edb9a x1851885432930880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099485.122314:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432930944:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122393:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099485.122439:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099485.122446:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432930816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122470:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099485.122529:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930944
00000100:00100000:0.0:1766099485.122583:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930816
00000100:00100000:0.0:1766099485.122609:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2472
00000100:00100000:0.0:1766099485.122617:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037d37a0f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432930944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122676:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037d37a0f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432930944:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (284us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099485.122686:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2472
00000100:00100000:0.0:1766099485.122711:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2473
00000100:00100000:0.0:1766099485.122714:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432930816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122739:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c44ad7a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432930816:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (269us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099485.122745:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2473
00000100:00100000:0.0:1766099485.122800:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432930816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122811:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432930944:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122861:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432930880:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099485.122892:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099485.122923:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432930880
00000100:00100000:0.0:1766099485.122945:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1530
00000100:00100000:3.0:1766099485.123298:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1677
00000100:00100000:3.0:1766099485.123304:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432930752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099485.123353:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432930752:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (3440us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099485.123360:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1677
00000100:00100000:1.0:1766099485.123627:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cad6b0f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432931008:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099485.123662:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099485.123698:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931008
00000100:00100000:1.0:1766099485.123727:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1102
00000100:00100000:1.0:1766099485.123730:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432931008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099485.123764:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432931008:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (103us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099485.123771:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1102
00010000:00080000:1.0:1766099485.123798:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000cad6b0f4 x1851885432931008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099492 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099485.123808:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cad6b0f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432931008:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099485.123939:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432930752:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099485.127622:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432930880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099485.127740:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099485.127859:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432930880:12345-0@lo:8:kworker/u8:2.0 Request processed in 4908us (4966us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099485.127871:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1530
00000100:00080000:2.0:1766099485.128175:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099485.128181:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099485.128188:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432930880:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099485.632105:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099486.653776:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099487.677713:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099488.701683:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099489.564673:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099489.564745:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099489.564859:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220416
00000100:00100000:0.0:1766099489.564930:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1678
00000100:00100000:0.0:1766099489.564939:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417220416:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099489.565042:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417220416:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 102us (381us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099489.565050:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1678
00000100:00100000:2.0:1766099489.565900:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220480
00000100:00100000:2.0:1766099489.565951:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2474
00000100:00100000:2.0:1766099489.565956:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417220480:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099489.567165:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417220480:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1201us (2412us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099489.567180:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2474
00000100:00100000:0.0:1766099489.570390:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099489.572036:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220544
00000100:00100000:3.0:1766099489.572115:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1531
00000100:00100000:3.0:1766099489.572124:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d84c111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417220544:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099489.572233:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099489.578015:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d84c111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417220544:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 5877us (7623us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099489.578038:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1531
00010000:00010000:0.0:1766099489.725866:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099490.237763:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099490.237773:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099490.237777:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099490.237788:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3993
00000100:00080000:2.0:1766099490.237793:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099490.237800:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099490.237828:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000092527c0a x1851885432931200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099490.237897:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000179aab07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432931328:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.237985:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099490.238030:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099490.238036:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432931072:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238068:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099490.238093:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092527c0a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432931200:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238115:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099490.238160:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c315443c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432931264:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238187:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099490.238204:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099490.238209:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee68381b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432931136:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238229:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099490.238265:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931264
00000100:00100000:2.0:1766099490.238313:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931136
00000100:00100000:2.0:1766099490.238338:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2475
00000100:00100000:2.0:1766099490.238344:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd789e54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432931264:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238399:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd789e54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432931264:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (211us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099490.238407:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2475
00000100:00100000:2.0:1766099490.238424:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2476
00000100:00100000:2.0:1766099490.238427:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8301247 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432931136:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238452:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8301247 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432931136:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (223us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099490.238455:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931328
00000100:00100000:2.0:1766099490.238457:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2476
00000100:00100000:3.0:1766099490.238496:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931200
00000100:00100000:2.0:1766099490.238513:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee68381b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432931136:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099490.238513:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103
00000100:00100000:3.0:1766099490.238517:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f745127a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432931328:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238524:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c315443c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432931264:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099490.238572:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f745127a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432931328:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (589us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099490.238576:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931072
00000100:00100000:3.0:1766099490.238579:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103
00000100:00100000:3.0:1766099490.238591:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1532
00000100:00100000:3.0:1766099490.238595:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d96f6de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432931200:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238600:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1679
00000100:00100000:2.0:1766099490.238603:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432931072:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099490.238628:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432931072:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (560us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099490.238634:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1679
00000100:00100000:2.0:1766099490.238674:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432931072:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766099490.238685:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000179aab07 x1851885432931328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099497 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099490.238693:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000179aab07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432931328:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766099490.238694:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099490.238725:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d96f6de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432931200:12345-0@lo:8:kworker/u8:3.0 Request processed in 129us (610us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099490.238732:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1532
00000100:00080000:2.0:1766099490.238753:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099490.238756:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099490.238760:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092527c0a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432931200:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766099490.749783:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099491.773782:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099492.797793:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099493.822332:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099494.687224:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099494.687343:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220736
00000100:00100000:3.0:1766099494.687412:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1533
00000100:00100000:3.0:1766099494.687421:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417220736:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099494.687586:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099494.693164:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417220736:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 5730us (5938us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099494.693186:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1533
00000100:00100000:1.0:1766099494.693373:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099494.693438:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099494.694702:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220608
00000100:00100000:2.0:1766099494.695346:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1680
00000100:00100000:2.0:1766099494.695358:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417220608:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099494.695507:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417220608:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 149us (2069us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099494.695516:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1680
00000100:00100000:3.0:1766099494.699652:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220672
00000100:00100000:3.0:1766099494.699848:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2477
00000100:00100000:3.0:1766099494.699855:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3830d7e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417220672:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099494.700280:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3830d7e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417220672:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 208us (6692us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099494.700291:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2477
00010000:00010000:2.0:1766099494.848025:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099495.357740:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099495.357750:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099495.357754:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1766099495.357757:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092527c0a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432931392:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766099495.357761:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3998
00000100:00080000:2.0:1766099495.357764:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099495.357771:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099495.357796:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000094c05093 x1851885432931520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099495.357853:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c315443c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432931584:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099495.357869:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099495.357890:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099495.357902:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099495.357907:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee68381b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432931648:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099495.357907:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099495.357910:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000179aab07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432931456:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099495.357930:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099495.357934:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099495.357953:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432931520:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099495.357977:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099495.358069:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931648
00000100:00100000:3.0:1766099495.358075:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931520
00000100:00100000:3.0:1766099495.358127:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104
00000100:00100000:1.0:1766099495.358132:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1534
00000100:00100000:3.0:1766099495.358134:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000faff0b8a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432931648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099495.358137:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018198959 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432931520:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099495.358192:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000faff0b8a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432931648:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (261us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099495.358199:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104
00000100:00100000:3.0:1766099495.358225:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931584
00010000:02020000:1.0:1766099495.358231:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099495.358250:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931456
00000100:00100000:3.0:1766099495.358268:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2478
00000100:00100000:3.0:1766099495.358271:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085cf7081 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432931584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099495.358304:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085cf7081 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432931584:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (413us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099495.358310:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2478
00000100:00100000:1.0:1766099495.358318:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018198959 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432931520:12345-0@lo:8:kworker/u8:3.0 Request processed in 180us (341us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099495.358323:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2479
00000100:00100000:1.0:1766099495.358324:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1534
00000100:00100000:3.0:1766099495.358326:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432931456:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099495.358360:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099495.358364:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099495.358368:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094c05093 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432931520:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099495.358508:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931392
00000100:00100000:0.0:1766099495.358532:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1681
00000100:00100000:0.0:1766099495.358536:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432931392:12345-0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766099495.358590:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ee68381b x1851885432931648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099502 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099495.358590:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adfcc04b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432931456:12345-0@lo:400:kworker/u8:3.0 Request processed in 264us (656us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099495.358597:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2479
00000100:00100000:1.0:1766099495.358602:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee68381b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432931648:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099495.358630:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000179aab07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432931456:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099495.358640:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c315443c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432931584:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099495.359728:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432931392:12345-0@lo:400:kworker/u8:3.0 Request processed in 1184us (1854us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099495.359741:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1681
00000100:00100000:1.0:1766099495.361130:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092527c0a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432931392:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099495.869736:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099496.893655:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099497.917659:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099498.942820:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099499.805984:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099499.806019:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099499.806057:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099499.806124:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220928
00000100:00100000:3.0:1766099499.806195:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1535
00000100:00100000:3.0:1766099499.806204:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abcb39cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417220928:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766099499.806281:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220864
00010000:02020000:3.0:1766099499.806308:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099499.806336:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2480
00000100:00100000:2.0:1766099499.806344:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c45bca0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417220864:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099499.806401:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abcb39cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417220928:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 195us (342us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099499.806411:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1535
00000100:00100000:2.0:1766099499.806420:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c45bca0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417220864:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 73us (399us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099499.806433:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2480
00000100:00100000:0.0:1766099499.806564:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220800
00000100:00100000:0.0:1766099499.806604:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1682
00000100:00100000:0.0:1766099499.806608:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417220800:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099499.806638:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417220800:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 29us (663us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099499.806644:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1682
00010000:00010000:2.0:1766099499.966021:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099500.477734:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099500.477742:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099500.477745:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099500.477753:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4003
00000100:00080000:0.0:1766099500.477757:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099500.477762:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099500.477812:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000ae936e0 x1851885432931840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099500.477895:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000ae936e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432931840:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099500.477964:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099500.478008:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931840
00000100:00100000:0.0:1766099500.478086:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1536
00000100:00100000:0.0:1766099500.478091:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad90c4a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432931840:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099500.478187:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099500.478238:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad90c4a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432931840:12345-0@lo:8:kworker/u8:2.0 Request processed in 145us (275us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099500.478246:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1536
00000100:00080000:1.0:1766099500.478279:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099500.478284:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099500.478288:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000ae936e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432931840:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099500.478323:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432931712:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099500.478354:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099500.478381:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099500.478386:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432931968:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099500.478406:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099500.478463:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931968
00000100:00100000:3.0:1766099500.478509:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432931776:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099500.478524:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1105
00000100:00100000:1.0:1766099500.478530:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432931968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099500.478543:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099500.478563:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099500.478569:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432931904:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099500.478581:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432931968:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (174us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099500.478596:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1105
00000100:00100000:3.0:1766099500.478596:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099500.478628:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931776
00000100:00100000:3.0:1766099500.478654:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931904
00000100:00100000:3.0:1766099500.478669:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2481
00000100:00100000:3.0:1766099500.478673:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001047d83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432931776:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099500.478705:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001047d83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432931776:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (163us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099500.478710:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2481
00000100:00100000:3.0:1766099500.478724:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2482
00000100:00100000:3.0:1766099500.478728:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c6056fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432931904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099500.478750:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c6056fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432931904:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (157us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099500.478755:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2482
00000100:00100000:3.0:1766099500.478783:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432931904:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099500.478794:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432931776:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766099500.479536:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c402e8c6 x1851885432931968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099507 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099500.479550:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432931968:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099500.480357:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432931712
00000100:00100000:0.0:1766099500.480407:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1683
00000100:00100000:0.0:1766099500.480411:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432931712:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099500.480661:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432931712:12345-0@lo:400:kworker/u8:2.0 Request processed in 248us (2306us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099500.480668:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1683
00000100:00100000:3.0:1766099500.480965:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432931712:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099500.996734:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099502.013802:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099503.038224:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099504.061700:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099504.926959:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099504.927148:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099504.927189:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099504.927776:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221056
00000100:00100000:2.0:1766099504.927829:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2483
00000100:00100000:2.0:1766099504.927835:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b735f288 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417221056:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099504.927908:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b735f288 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417221056:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (955us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099504.927915:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2483
00000100:00100000:0.0:1766099504.928913:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417220992
00000100:00100000:0.0:1766099504.928959:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1684
00000100:00100000:0.0:1766099504.928964:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417220992:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099504.929008:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417220992:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 44us (1820us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099504.929015:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1684
00000100:00100000:3.0:1766099504.929984:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221120
00000100:00100000:0.0:1766099504.931170:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1537
00000100:00100000:0.0:1766099504.931179:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034f8339a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417221120:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099504.931277:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099504.931340:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034f8339a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417221120:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 160us (4192us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099504.931347:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1537
00010000:00010000:0.0:1766099505.085794:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099505.597819:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099505.597827:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099505.597830:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099505.597839:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4008
00000100:00080000:2.0:1766099505.597842:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099505.597847:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099505.597867:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000024a81cfa x1851885432932160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099505.597912:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432932224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099505.597970:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099505.598122:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432932032:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099505.598155:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099505.598182:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099505.598186:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432932288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099505.598209:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099505.598231:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000388d2359 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432932096:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099505.598240:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932288
00000100:00100000:3.0:1766099505.598278:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1106
00000100:00100000:3.0:1766099505.598284:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0a9bb15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432932288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099505.598348:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0a9bb15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432932288:12345-0@lo:400:kworker/u8:3.0 Request processed in 63us (138us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099505.598356:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1106
00000100:00100000:3.0:1766099505.598379:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932032
00000100:00100000:3.0:1766099505.598401:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1685
00000100:00100000:3.0:1766099505.598405:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432932032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099505.598445:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432932160:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099505.598445:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432932032:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (290us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099505.598455:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1685
00000100:00100000:1.0:1766099505.598496:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766099505.598517:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000132c823c x1851885432932288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099512 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099505.598520:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932160
00000100:00100000:3.0:1766099505.598529:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432932288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099505.598537:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432932032:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099505.598538:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1538
00000100:00100000:1.0:1766099505.598541:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432932160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099505.598629:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099505.598681:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432932160:12345-0@lo:8:kworker/u8:3.0 Request processed in 137us (184us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099505.598687:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1538
00000100:00100000:1.0:1766099505.598729:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932224
00000100:00100000:1.0:1766099505.598762:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2484
00000100:00100000:1.0:1766099505.598766:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f11e586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432932224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099505.598798:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f11e586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432932224:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (829us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099505.598799:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099505.598804:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2484
00000100:00100000:1.0:1766099505.598814:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932096
00000100:00100000:1.0:1766099505.598830:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2485
00000100:00100000:1.0:1766099505.598834:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6a8540f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432932096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099505.598863:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6a8540f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432932096:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (64us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099505.598869:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2485
00000100:00100000:1.0:1766099505.598893:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000388d2359 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432932096:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099505.599560:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432932224:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099505.601445:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099505.601466:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099505.601473:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432932160:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766099506.110726:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099507.134642:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099508.157781:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099509.183156:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099510.048126:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099510.048229:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099510.048313:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221312
00000100:00100000:2.0:1766099510.048364:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1539
00000100:00100000:2.0:1766099510.048371:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417221312:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099510.048512:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099510.048571:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417221312:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 200us (452us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099510.048579:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1539
00000100:00100000:1.0:1766099510.049079:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099510.049462:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221248
00000100:00100000:3.0:1766099510.049508:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2486
00000100:00100000:3.0:1766099510.049512:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019993a78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417221248:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099510.049558:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019993a78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417221248:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 45us (1330us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099510.049564:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2486
00000100:00100000:2.0:1766099510.050090:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221184
00000100:00100000:2.0:1766099510.050125:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1686
00000100:00100000:2.0:1766099510.050130:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221184:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099510.050156:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221184:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 26us (1079us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099510.050161:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1686
00010000:00010000:2.0:1766099510.220153:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099510.717806:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099510.717814:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099510.717817:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099510.717825:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4014
00000100:00080000:0.0:1766099510.717828:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099510.717833:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099510.717853:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000871971ee x1851885432932480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099510.717910:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077a1e307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432932416:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099510.717972:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099510.718007:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099510.718012:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d287ef1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432932544:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099510.718033:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099510.718063:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932416
00000100:00100000:3.0:1766099510.718131:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432932352:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718166:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099510.718187:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099510.718192:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000585c6915 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432932608:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718216:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099510.718240:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932352
00000100:00100000:3.0:1766099510.718275:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1687
00000100:00100000:3.0:1766099510.718280:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432932352:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718321:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432932352:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (156us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099510.718329:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1687
00000100:00100000:3.0:1766099510.718356:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932608
00000100:00100000:3.0:1766099510.718375:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1107
00000100:00100000:3.0:1766099510.718390:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c8bf3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432932608:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718418:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c8bf3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432932608:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (202us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099510.718424:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1107
00010000:00080000:3.0:1766099510.718452:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000585c6915 x1851885432932608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099517 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099510.718463:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000585c6915 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432932608:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718472:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432932352:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718524:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932544
00000100:00100000:3.0:1766099510.718551:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2487
00000100:00100000:3.0:1766099510.718554:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eb9fd8f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432932544:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718586:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eb9fd8f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432932544:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (552us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099510.718591:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2487
00000100:00100000:3.0:1766099510.718608:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432932480:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718630:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099510.718652:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932480
00000100:00100000:3.0:1766099510.718669:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1540
00000100:00100000:3.0:1766099510.718672:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432932480:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099510.718758:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099510.718789:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432932480:12345-0@lo:8:kworker/u8:3.0 Request processed in 116us (159us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099510.718796:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1540
00000100:00080000:3.0:1766099510.718817:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099510.718820:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099510.718823:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432932480:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099510.718865:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d287ef1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432932544:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099510.719449:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2488
00000100:00100000:0.0:1766099510.719453:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cabff9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432932416:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099510.719502:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cabff9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432932416:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (1531us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099510.719507:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2488
00000100:00100000:0.0:1766099510.719532:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077a1e307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432932416:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099511.229686:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099512.257377:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099513.277683:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099514.302236:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099515.167813:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099515.169210:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221376
00000100:00100000:0.0:1766099515.169266:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1688
00000100:00100000:0.0:1766099515.169272:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417221376:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099515.169354:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417221376:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 81us (1549us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099515.169362:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1688
00000100:00100000:2.0:1766099515.171705:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099515.175639:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221440
00000100:00100000:3.0:1766099515.175703:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2489
00000100:00100000:3.0:1766099515.175710:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417221440:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099515.175801:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417221440:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 90us (4102us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099515.175809:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2489
00000100:00100000:1.0:1766099515.176210:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099515.178355:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221504
00000100:00100000:3.0:1766099515.180817:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1541
00000100:00100000:3.0:1766099515.180827:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff29ec37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417221504:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099515.180941:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099515.181018:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff29ec37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417221504:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 191us (4810us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099515.181026:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1541
00010000:00010000:2.0:1766099515.325759:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099515.837745:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099515.837752:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099515.837755:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099515.837762:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4019
00000100:00080000:2.0:1766099515.837766:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099515.837770:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099515.837792:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000018198959 x1851885432932800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099515.837837:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf6d59f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432932864:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.837900:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099515.837933:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099515.837938:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432932736:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.837963:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099515.837994:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932864
00000100:00100000:2.0:1766099515.838030:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932736
00000100:00100000:2.0:1766099515.838048:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2490
00000100:00100000:2.0:1766099515.838053:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432932864:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.838097:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432932864:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (197us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099515.838104:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2490
00000100:00100000:2.0:1766099515.838119:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2491
00000100:00100000:2.0:1766099515.838122:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432932736:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.838146:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432932736:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (182us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099515.838151:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2491
00000100:00100000:2.0:1766099515.838196:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432932736:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.838206:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf6d59f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432932864:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099515.838449:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432932928:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099515.838500:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099515.838527:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099515.838532:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432932672:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099515.838557:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099515.838629:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932928
00000100:00100000:2.0:1766099515.838653:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1108
00000100:00100000:2.0:1766099515.838656:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077a1e307 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432932928:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.838687:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077a1e307 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432932928:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (187us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099515.838692:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1108
00010000:00080000:0.0:1766099515.840081:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000024a81cfa x1851885432932928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099522 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099515.840094:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432932928:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099515.840109:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932672
00000100:00100000:1.0:1766099515.840150:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1689
00000100:00100000:1.0:1766099515.840154:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432932672:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099515.840199:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432932672:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (1642us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099515.840204:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1689
00000100:00100000:0.0:1766099515.841821:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432932672:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099515.841931:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432932800:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099515.841983:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099515.844066:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932800
00000100:00100000:0.0:1766099515.844130:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1542
00000100:00100000:0.0:1766099515.844136:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432932800:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099515.844231:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099515.844300:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432932800:12345-0@lo:8:kworker/u8:2.0 Request processed in 163us (2317us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099515.844307:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1542
00000100:00080000:2.0:1766099515.844775:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099515.844779:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099515.844784:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432932800:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099516.352251:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099517.374303:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099518.397846:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099519.422553:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099520.287571:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099520.287635:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221568
00000100:00100000:2.0:1766099520.287691:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1690
00000100:00100000:2.0:1766099520.287698:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221568:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099520.287779:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221568:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 80us (215us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099520.287786:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1690
00000100:00100000:0.0:1766099520.292958:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099520.293029:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221632
00000100:00100000:3.0:1766099520.293091:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2492
00000100:00100000:3.0:1766099520.293098:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d8990d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417221632:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099520.293179:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d8990d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417221632:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 80us (229us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099520.293187:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2492
00000100:00100000:0.0:1766099520.293847:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099520.293983:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221696
00000100:00100000:3.0:1766099520.294020:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1543
00000100:00100000:3.0:1766099520.294025:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c41f2c96 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417221696:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099520.294118:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099520.294168:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c41f2c96 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417221696:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 142us (323us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099520.294176:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1543
00010000:00010000:0.0:1766099520.447732:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099520.957709:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766099520.957752:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099520.957758:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099520.957762:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099520.957768:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4024
00000100:00080000:2.0:1766099520.957772:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099520.957777:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099520.957798:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000132c823c x1851885432933120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099520.957910:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432933184:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099520.957969:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099520.957995:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099520.958000:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432933056:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099520.958017:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099520.958049:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933184
00000100:00100000:3.0:1766099520.958085:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933056
00000100:00100000:3.0:1766099520.958105:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2493
00000100:00100000:3.0:1766099520.958111:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432933184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099520.958154:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432933184:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (185us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099520.958358:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432933248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958391:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099520.958408:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432932992:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958433:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099520.958466:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2494
00000100:00100000:1.0:1766099520.958470:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007bef605c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432933056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958522:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007bef605c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432933056:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (504us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099520.958529:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2494
00000100:00100000:1.0:1766099520.958781:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432933056:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958792:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432933184:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958816:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933248
00000100:00100000:1.0:1766099520.958841:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109
00000100:00100000:1.0:1766099520.958845:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432933248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958880:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432933248:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (489us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099520.958886:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109
00010000:00080000:1.0:1766099520.958908:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000024a81cfa x1851885432933248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099527 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099520.958917:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432933248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958943:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432932992
00000100:00100000:1.0:1766099520.958966:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1691
00000100:00100000:1.0:1766099520.958970:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432932992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.958999:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432932992:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (566us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099520.959006:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1691
00000100:00100000:1.0:1766099520.959030:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432933120:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099520.959050:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099520.959081:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018198959 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432932992:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099520.959096:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933120
00000100:00100000:1.0:1766099520.959111:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1544
00000100:00100000:1.0:1766099520.959114:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e67a7300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432933120:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099520.959977:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2493
00010000:02020000:1.0:1766099520.961454:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099520.961557:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e67a7300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432933120:12345-0@lo:8:kworker/u8:2.0 Request processed in 2439us (2504us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099520.961567:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1544
00000100:00080000:2.0:1766099520.962135:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099520.962140:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099520.962145:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000132c823c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432933120:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099521.470419:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099522.493718:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099523.519150:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099524.541714:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099525.405124:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099525.405898:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221888
00000100:00100000:2.0:1766099525.406014:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1545
00000100:00100000:2.0:1766099525.406021:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016c31705 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417221888:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099525.406118:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099525.406199:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016c31705 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417221888:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 177us (1081us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099525.406207:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1545
00000100:00100000:3.0:1766099525.406644:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099525.406687:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221760
00000100:00100000:3.0:1766099525.406765:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099525.406806:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221824
00000100:00100000:3.0:1766099525.406853:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2495
00000100:00100000:3.0:1766099525.406860:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417221824:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099525.406930:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417221824:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 69us (166us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099525.406937:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2495
00000100:00100000:2.0:1766099525.407038:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1692
00000100:00100000:2.0:1766099525.407045:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221760:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099525.407102:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221760:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 57us (467us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099525.407109:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1692
00010000:00010000:2.0:1766099525.566446:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099526.082448:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099526.082458:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099526.082462:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099526.082473:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4029
00000100:00100000:1.0:1766099526.082617:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099526.082643:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432933376:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099526.082735:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099526.082779:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099526.082788:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9fe64a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432933312:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099526.082820:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099526.082858:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933312
00000100:00100000:1.0:1766099526.082914:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1693
00000100:00100000:1.0:1766099526.082921:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432933312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099526.082976:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432933312:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (155us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099526.082984:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1693
00000100:00100000:1.0:1766099526.083040:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9fe64a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432933312:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099526.083089:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933376
00000100:00100000:1.0:1766099526.083118:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2496
00000100:00100000:1.0:1766099526.083122:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1197d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432933376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099526.083160:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1197d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432933376:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (427us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099526.083167:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2496
00000100:00100000:1.0:1766099526.083194:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432933376:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099526.083270:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099526.083275:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099526.083306:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000077d8956b x1851885432933440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099526.083372:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0a9bb15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432933504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099526.083401:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099526.083424:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c8bf3c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432933568:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099526.083448:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099526.083496:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933568
00000100:00100000:2.0:1766099526.083961:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933504
00000100:00100000:2.0:1766099526.083990:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2497
00000100:00100000:2.0:1766099526.083995:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432933504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099526.084036:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432933504:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (635us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099526.084041:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2497
00000100:00100000:2.0:1766099526.084070:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0a9bb15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432933504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099526.084324:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1110
00000100:00100000:3.0:1766099526.084329:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432933568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099526.084367:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432933568:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (918us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099526.084373:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1110
00010000:00080000:3.0:1766099526.084401:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001c8bf3c7 x1851885432933568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099533 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099526.084415:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c8bf3c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432933568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099526.086200:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077d8956b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432933440:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099526.086302:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099526.086931:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933440
00000100:00100000:2.0:1766099526.086977:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1546
00000100:00100000:2.0:1766099526.086983:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eaa17f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432933440:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099526.087080:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099526.087142:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eaa17f27 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432933440:12345-0@lo:8:kworker/u8:3.0 Request processed in 158us (842us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099526.087148:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1546
00000100:00080000:2.0:1766099526.087188:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099526.087193:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099526.087197:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077d8956b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432933440:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099526.595605:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099527.614527:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099528.637648:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099529.664038:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099530.524629:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099530.524757:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222080
00000100:00100000:2.0:1766099530.524820:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1547
00000100:00100000:2.0:1766099530.524828:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c8bf3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417222080:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099530.524932:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099530.525006:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c8bf3c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417222080:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 178us (389us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099530.525012:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1547
00000100:00100000:3.0:1766099530.526988:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099530.527041:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099530.527145:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222016
00000100:00100000:2.0:1766099530.527212:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2498
00000100:00100000:2.0:1766099530.527220:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042e452f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417222016:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099530.527325:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042e452f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417222016:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 104us (283us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099530.527333:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2498
00000100:00100000:1.0:1766099530.527765:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417221952
00000100:00100000:1.0:1766099530.527807:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1694
00000100:00100000:1.0:1766099530.527812:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221952:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099530.527854:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417221952:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 42us (878us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099530.527862:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1694
00010000:00010000:2.0:1766099530.688549:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099531.198362:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099531.198372:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099531.198377:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099531.198387:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4034
00000100:00080000:1.0:1766099531.198391:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099531.198398:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099531.198427:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f13f01a7 x1851885432933760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099531.198511:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432933696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099531.198604:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099531.198646:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099531.198652:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d4f110b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432933824:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099531.198694:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099531.198776:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f13f01a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432933760:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099531.198810:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099531.198864:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933696
00000100:00100000:2.0:1766099531.198910:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933824
00000100:00100000:2.0:1766099531.198928:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2499
00000100:00100000:2.0:1766099531.198934:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066267589 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432933696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099531.198987:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066267589 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432933696:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (385us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099531.198995:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2499
00000100:00100000:2.0:1766099531.199012:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2500
00000100:00100000:2.0:1766099531.199015:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432933824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099531.199038:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432933824:12345-0@lo:400:kworker/u8:3.0 Request processed in 21us (344us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099531.199043:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2500
00000100:00100000:3.0:1766099531.199310:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432933888:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099531.199344:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099531.199375:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099531.199381:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048d0dc2f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432933632:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099531.199412:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099531.199444:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933632
00000100:00100000:3.0:1766099531.199474:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1695
00000100:00100000:3.0:1766099531.199500:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432933632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099531.199535:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432933632:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (123us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099531.199541:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1695
00000100:00100000:3.0:1766099531.199590:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048d0dc2f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432933632:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099531.199664:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933760
00000100:00100000:2.0:1766099531.199709:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933888
00000100:00100000:2.0:1766099531.199731:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1111
00000100:00100000:2.0:1766099531.199735:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091b162e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432933888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099531.199773:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091b162e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432933888:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (428us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099531.199779:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1111
00000100:00100000:2.0:1766099531.199792:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1548
00000100:00100000:2.0:1766099531.199795:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432933760:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099531.199906:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099531.199942:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432933760:12345-0@lo:8:kworker/u8:3.0 Request processed in 145us (1131us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099531.199948:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1548
00000100:00080000:2.0:1766099531.199977:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099531.199981:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099531.199987:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f13f01a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432933760:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099531.200166:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d4f110b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432933824:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099531.200178:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432933696:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099531.200790:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000be044d66 x1851885432933888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099538 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099531.200803:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432933888:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099531.709722:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099532.735387:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099533.762347:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099534.782245:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099535.645445:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099535.645550:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099535.645668:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222208
00000100:00100000:0.0:1766099535.645678:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222144
00000100:00100000:0.0:1766099535.645741:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1696
00000100:00100000:1.0:1766099535.645746:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2501
00000100:00100000:0.0:1766099535.645753:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417222144:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099535.645756:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009470f4e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417222208:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099535.645852:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417222144:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 101us (303us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099535.645861:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1696
00000100:00100000:1.0:1766099535.646120:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009470f4e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417222208:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 364us (685us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099535.646128:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2501
00000100:00100000:2.0:1766099535.651617:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099535.651761:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222272
00000100:00100000:0.0:1766099535.652369:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1549
00000100:00100000:0.0:1766099535.652380:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018198959 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417222272:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099535.652513:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099535.652622:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018198959 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417222272:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 242us (1016us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099535.652631:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1549
00010000:00010000:0.0:1766099535.805824:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099536.317786:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099536.317795:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099536.317799:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099536.317808:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4039
00000100:00080000:2.0:1766099536.317812:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099536.317819:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099536.317843:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ce45f83d x1851885432934080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099536.317878:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432934016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099536.317960:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099536.318013:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934016
00000100:00100000:3.0:1766099536.318130:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099536.318139:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432933952:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099536.318177:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099536.318233:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432933952
00000100:00100000:3.0:1766099536.318259:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1697
00000100:00100000:3.0:1766099536.318266:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432933952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099536.318310:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432933952:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (133us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099536.318316:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1697
00000100:00100000:3.0:1766099536.318357:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432933952:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099536.319372:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432934080:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099536.319423:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099536.319498:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2502
00000100:00100000:1.0:1766099536.319504:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432934016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099536.319557:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432934016:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (1598us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099536.319564:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2502
00000100:00100000:1.0:1766099536.319605:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432934016:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099536.319649:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934080
00000100:00100000:1.0:1766099536.319672:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1550
00000100:00100000:1.0:1766099536.319676:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46034f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432934080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099536.319775:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099536.319917:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432934144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099536.319950:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099536.320021:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934144
00000100:00100000:2.0:1766099536.320049:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2503
00000100:00100000:2.0:1766099536.320053:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432934144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099536.320108:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432934144:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (158us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099536.320114:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2503
00000100:00100000:2.0:1766099536.320138:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432934144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099536.320154:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432934208:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099536.320185:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099536.320224:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934208
00000100:00100000:2.0:1766099536.320247:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112
00000100:00100000:2.0:1766099536.320251:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003dc105a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432934208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099536.320282:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003dc105a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432934208:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (98us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099536.320288:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112
00010000:00080000:2.0:1766099536.320316:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000024a81cfa x1851885432934208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099543 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099536.320326:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432934208:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099536.320440:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46034f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432934080:12345-0@lo:8:kworker/u8:2.0 Request processed in 762us (1016us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099536.320447:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1550
00000100:00080000:3.0:1766099536.320574:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099536.320578:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099536.320583:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432934080:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099536.831871:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099537.853808:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099538.877749:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099539.906095:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099540.769614:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099540.769678:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099540.769737:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099540.769874:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222336
00000100:00100000:2.0:1766099540.769947:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1698
00000100:00100000:2.0:1766099540.769956:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417222336:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099540.770047:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417222336:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 91us (369us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099540.770055:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1698
00000100:00100000:1.0:1766099540.770368:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222400
00000100:00100000:1.0:1766099540.770415:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2504
00000100:00100000:1.0:1766099540.770420:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093648709 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417222400:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099540.770474:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093648709 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417222400:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 53us (873us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099540.770511:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2504
00000100:00100000:3.0:1766099540.771224:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222464
00000100:00100000:3.0:1766099540.771269:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1551
00000100:00100000:3.0:1766099540.771274:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014e0e83d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417222464:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099540.771377:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099540.774023:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014e0e83d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417222464:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 2736us (4274us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099540.774043:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1551
00010000:00010000:0.0:1766099540.925842:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099541.438408:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099541.438418:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099541.438423:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099541.438433:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4044
00000100:00080000:0.0:1766099541.438437:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099541.438454:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099541.438497:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ef33bb80 x1851885432934400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099541.438868:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c6fd5dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432934528:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099541.438954:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099541.439936:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000669f53de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432934336:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.439997:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099541.440044:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099541.440051:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432934464:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.440079:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099541.440124:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934336
00000100:00100000:1.0:1766099541.440186:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934464
00000100:00100000:1.0:1766099541.440208:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2505
00000100:00100000:1.0:1766099541.440215:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432934336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.440272:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432934336:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (274us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099541.440280:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2505
00000100:00100000:1.0:1766099541.440311:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2506
00000100:00100000:1.0:1766099541.440315:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090ca65cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432934464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.440347:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090ca65cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432934464:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (268us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099541.440353:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2506
00000100:00100000:1.0:1766099541.440374:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000030a9820 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432934272:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.440404:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099541.440457:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432934464:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.440496:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000669f53de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432934336:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099541.440911:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ef33bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432934400:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099541.441613:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934528
00000100:00100000:0.0:1766099541.444055:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934272
00000100:00100000:0.0:1766099541.444123:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1699
00000100:00100000:0.0:1766099541.444130:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432934272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099541.444256:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432934272:12345-0@lo:400:kworker/u8:3.0 Request processed in 125us (3851us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099541.444265:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1699
00000100:00100000:0.0:1766099541.444355:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113
00000100:00100000:0.0:1766099541.444359:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432934528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099541.444394:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432934528:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (5441us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099541.444400:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113
00000100:00100000:0.0:1766099541.444452:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000030a9820 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432934272:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099541.444862:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766099541.444920:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002c6fd5dd x1851885432934528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099548 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099541.444933:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c6fd5dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432934528:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099541.444996:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934400
00000100:00100000:1.0:1766099541.445025:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1552
00000100:00100000:1.0:1766099541.445029:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000347bd79f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432934400:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099541.445133:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099541.445184:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000347bd79f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432934400:12345-0@lo:8:kworker/u8:3.0 Request processed in 154us (324us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099541.445189:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1552
00000100:00080000:1.0:1766099541.445213:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099541.445218:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099541.445222:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ef33bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432934400:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766099541.953421:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099542.973938:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099543.997788:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099545.021741:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099545.885707:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099545.885756:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099545.885798:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099545.885907:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222528
00000100:00100000:0.0:1766099545.885968:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1700
00000100:00100000:0.0:1766099545.885975:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417222528:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099545.886046:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417222528:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 70us (247us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099545.886053:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1700
00000100:00100000:1.0:1766099545.886542:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222656
00000100:00100000:1.0:1766099545.886577:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1553
00000100:00100000:1.0:1766099545.886581:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417222656:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099545.886684:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099545.887013:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222592
00000100:00100000:3.0:1766099545.887611:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2507
00000100:00100000:3.0:1766099545.887617:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000411dc2e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417222592:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099545.887814:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417222656:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1231us (2057us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099545.887823:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1553
00000100:00100000:3.0:1766099545.890206:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000411dc2e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417222592:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2580us (4500us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099545.890221:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2507
00010000:00010000:2.0:1766099546.045750:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099546.559281:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099546.559290:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099546.559294:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099546.559303:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4049
00000100:00080000:3.0:1766099546.559307:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099546.559313:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099546.559335:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001415cdc3 x1851885432934720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099546.559502:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a02de30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432934848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099546.559545:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9fe64a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432934784:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099546.559585:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099546.559617:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099546.559621:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099546.559633:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077d8956b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432934592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099546.559656:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099546.559666:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0a9bb15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432934656:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099546.559679:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099546.559710:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099546.559738:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934592
00000100:00100000:3.0:1766099546.559759:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934784
00000100:00100000:2.0:1766099546.559791:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1701
00000100:00100000:2.0:1766099546.559800:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432934592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099546.559810:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934656
00000100:00100000:3.0:1766099546.559836:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2508
00000100:00100000:3.0:1766099546.559843:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c0fbdcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432934784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099546.559859:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432934592:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (182us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099546.559868:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1701
00000100:00100000:3.0:1766099546.559908:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c0fbdcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432934784:12345-0@lo:400:kworker/u8:3.0 Request processed in 63us (293us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099546.559917:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2508
00000100:00100000:2.0:1766099546.559923:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077d8956b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432934592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099546.559941:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2509
00000100:00100000:3.0:1766099546.559947:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d65d7291 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432934656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099546.559982:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9fe64a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432934784:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099546.559989:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d65d7291 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432934656:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (280us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099546.559998:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2509
00000100:00100000:2.0:1766099546.560015:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0a9bb15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432934656:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099546.560207:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432934720:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099546.560243:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099546.560287:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934848
00000100:00100000:1.0:1766099546.560320:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934720
00000100:00100000:1.0:1766099546.560336:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1114
00000100:00100000:1.0:1766099546.560340:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432934848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099546.560371:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432934848:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (792us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099546.560380:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1114
00000100:00100000:1.0:1766099546.560391:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1554
00000100:00100000:1.0:1766099546.560394:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be044d66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432934720:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099546.560527:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099546.560556:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be044d66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432934720:12345-0@lo:8:kworker/u8:3.0 Request processed in 161us (313us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099546.560562:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1554
00000100:00080000:1.0:1766099546.560599:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099546.560603:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099546.560607:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432934720:0@lo:8:kworker/u8:3.0
00010000:00080000:1.0:1766099546.560633:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000a02de30 x1851885432934848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099553 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099546.560643:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a02de30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432934848:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099547.069710:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099548.093768:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099549.117723:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099550.141757:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099551.006560:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099551.006612:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099551.006730:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222848
00000100:00100000:3.0:1766099551.006794:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1555
00000100:00100000:3.0:1766099551.006802:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000030a9820 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417222848:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099551.006904:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099551.006974:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000030a9820 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417222848:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 172us (424us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099551.006980:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1555
00000100:00100000:2.0:1766099551.008222:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222720
00000100:00100000:2.0:1766099551.008286:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1702
00000100:00100000:2.0:1766099551.008292:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417222720:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099551.008352:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417222720:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 60us (1740us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099551.008359:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1702
00000100:00100000:0.0:1766099551.010972:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099551.011227:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222784
00000100:00100000:2.0:1766099551.011289:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2510
00000100:00100000:2.0:1766099551.011296:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dad21f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417222784:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099551.011380:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dad21f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417222784:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 84us (418us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099551.011388:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2510
00010000:00010000:0.0:1766099551.168503:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099551.680565:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe958a22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432934912:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099551.680573:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099551.680581:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099551.680584:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099551.680593:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4055
00000100:00080000:0.0:1766099551.680597:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099551.680604:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099551.680628:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e96bbcf1 x1851885432935040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099551.680680:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099551.680706:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000caf250b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432934976:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099551.680728:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099551.680740:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ef33bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432935168:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099551.680780:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099551.680788:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099551.680829:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099551.680842:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432935040:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099551.680858:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7f92201 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432935104:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099551.680875:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099551.680904:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099551.680935:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934912
00000100:00100000:0.0:1766099551.680955:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432934976
00000100:00100000:1.0:1766099551.680992:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1703
00000100:00100000:1.0:1766099551.681001:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432934912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099551.681005:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935104
00000100:00100000:0.0:1766099551.681036:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2511
00000100:00100000:0.0:1766099551.681043:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432934976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099551.681082:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432934912:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (395us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099551.681093:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1703
00000100:00100000:0.0:1766099551.681118:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432934976:12345-0@lo:400:kworker/u8:2.0 Request processed in 73us (338us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099551.681131:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2511
00000100:00100000:1.0:1766099551.681148:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935168
00000100:00100000:0.0:1766099551.681159:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2512
00000100:00100000:0.0:1766099551.681164:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000669f53de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432935104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099551.681183:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935040
00000100:00100000:0.0:1766099551.681212:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000669f53de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432935104:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (306us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099551.681217:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1115
00000100:00100000:0.0:1766099551.681227:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2512
00000100:00100000:1.0:1766099551.681228:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f13f01a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432935168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099551.681278:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f13f01a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432935168:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (492us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099551.681285:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7f92201 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432935104:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099551.681300:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000caf250b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432934976:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099551.681343:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1556
00000100:00100000:0.0:1766099551.681348:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d0dc2f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432935040:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099551.681445:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:2.0:1766099551.681460:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ef33bb80 x1851885432935168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099558 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099551.681506:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ef33bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432935168:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099551.681528:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe958a22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432934912:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099551.681539:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d0dc2f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432935040:12345-0@lo:8:kworker/u8:2.0 Request processed in 184us (661us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099551.681549:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1556
00000100:00080000:2.0:1766099551.681591:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099551.681597:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099551.681601:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432935040:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099551.681678:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1115
00010000:00010000:2.0:1766099552.191023:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099553.214876:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099554.242019:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099555.264605:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099556.127619:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099556.127677:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099556.127772:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099556.127870:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223040
00000100:00100000:1.0:1766099556.127930:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1557
00000100:00100000:1.0:1766099556.127937:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e82385d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417223040:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099556.128070:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099556.128147:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e82385d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417223040:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 210us (540us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099556.128155:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1557
00000100:00100000:0.0:1766099556.128966:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222976
00000100:00100000:0.0:1766099556.129028:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2513
00000100:00100000:0.0:1766099556.129034:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a02de30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417222976:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.129108:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a02de30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417222976:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 73us (1337us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099556.129115:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2513
00000100:00100000:2.0:1766099556.129205:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417222912
00000100:00100000:2.0:1766099556.129243:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1704
00000100:00100000:2.0:1766099556.129248:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417222912:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099556.129295:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417222912:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 46us (1618us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099556.129302:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1704
00010000:00010000:0.0:1766099556.287225:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099556.799443:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099556.799454:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099556.799459:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099556.799469:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4060
00000100:00080000:0.0:1766099556.799474:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099556.799502:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099556.799531:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000cde0997c x1851885432935360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099556.799602:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7f92201 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432935296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.799686:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099556.799727:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099556.799733:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad90c4a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432935424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.799756:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099556.799800:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935296
00000100:00100000:0.0:1766099556.799867:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935424
00000100:00100000:0.0:1766099556.799923:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2514
00000100:00100000:0.0:1766099556.799930:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432935296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.799988:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432935296:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (303us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099556.799996:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2514
00000100:00100000:0.0:1766099556.800018:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2515
00000100:00100000:0.0:1766099556.800021:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f484c134 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432935424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.800062:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f484c134 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432935424:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (305us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099556.800068:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2515
00000100:00100000:0.0:1766099556.800130:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad90c4a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432935424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.800141:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7f92201 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432935296:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099556.800543:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432935360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099556.800606:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099556.801301:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000caf250b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432935232:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099556.801354:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099556.801395:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099556.801402:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d287ef1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432935488:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099556.801428:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099556.802274:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935360
00000100:00100000:2.0:1766099556.802274:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935232
00000100:00100000:2.0:1766099556.802320:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1705
00000100:00100000:1.0:1766099556.802325:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935488
00000100:00100000:2.0:1766099556.802327:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432935232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099556.802350:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116
00000100:00100000:1.0:1766099556.802353:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001415cdc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432935488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099556.803804:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1558
00000100:00100000:3.0:1766099556.803817:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432935360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099556.803932:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099556.804869:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001415cdc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432935488:12345-0@lo:400:kworker/u8:3.0 Request processed in 2509us (3435us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099556.804883:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116
00000100:00100000:2.0:1766099556.805138:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432935232:12345-0@lo:400:kworker/u8:3.0 Request processed in 2808us (3783us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099556.805149:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1705
00000100:00100000:3.0:1766099556.806213:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432935360:12345-0@lo:8:kworker/u8:3.0 Request processed in 2390us (5601us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099556.806226:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1558
00000100:00080000:2.0:1766099556.806441:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099556.806447:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099556.806453:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432935360:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766099556.806525:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002d287ef1 x1851885432935488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099563 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099556.806545:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d287ef1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432935488:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099556.806556:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000caf250b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432935232:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099557.309804:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099558.333709:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099559.357839:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099560.381787:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099561.244698:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099561.245083:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223232
00000100:00100000:2.0:1766099561.245147:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1559
00000100:00100000:2.0:1766099561.245156:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0a9bb15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417223232:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099561.245272:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099561.245286:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099561.245366:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0a9bb15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417223232:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 210us (678us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099561.245376:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1559
00000100:00100000:0.0:1766099561.245390:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099561.245425:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223168
00000100:00100000:2.0:1766099561.245468:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2516
00000100:00100000:2.0:1766099561.245512:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000caf250b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417223168:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099561.245560:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223104
00000100:00100000:2.0:1766099561.245579:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000caf250b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417223168:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 104us (191us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099561.245587:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2516
00000100:00100000:1.0:1766099561.245596:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1706
00000100:00100000:1.0:1766099561.245602:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417223104:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099561.245633:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417223104:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 31us (349us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099561.245640:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1706
00010000:00010000:2.0:1766099561.405978:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099561.917777:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099561.917786:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099561.917791:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099561.917801:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4065
00000100:00080000:2.0:1766099561.917806:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099561.917813:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099561.917840:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ce45f83d x1851885432935680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099561.917889:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432935680:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099561.917974:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432935616:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099561.917975:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099561.918040:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099561.918047:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935680
00000100:00100000:0.0:1766099561.918061:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099561.918067:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432935744:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918088:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099561.918089:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1560
00000100:00100000:2.0:1766099561.918096:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432935680:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918117:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935616
00000100:00100000:0.0:1766099561.918147:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935744
00000100:00100000:0.0:1766099561.918164:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2517
00000100:00100000:0.0:1766099561.918167:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d287ef1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432935616:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:2.0:1766099561.918192:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099561.918214:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d287ef1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432935616:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (173us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099561.918219:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ebe6253 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432935680:12345-0@lo:8:kworker/u8:2.0 Request processed in 124us (248us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099561.918221:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2517
00000100:00100000:2.0:1766099561.918225:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1560
00000100:00100000:0.0:1766099561.918239:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2518
00000100:00100000:0.0:1766099561.918242:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7f92201 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432935744:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918265:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7f92201 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432935744:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (177us total) trans 0 rc 0/0
00000100:00080000:2.0:1766099561.918267:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:0.0:1766099561.918270:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2518
00000100:00080000:2.0:1766099561.918270:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099561.918274:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432935680:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918294:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432935552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918322:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099561.918351:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099561.918356:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432935808:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918377:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099561.918400:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935808
00000100:00100000:0.0:1766099561.918404:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432935744:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099561.918412:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432935616:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099561.918430:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1117
00000100:00100000:1.0:1766099561.918433:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad90c4a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432935808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099561.918470:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad90c4a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432935808:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (93us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099561.918498:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1117
00010000:00080000:1.0:1766099561.918528:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000612198b4 x1851885432935808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099568 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099561.918536:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432935808:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099561.920883:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935552
00000100:00100000:2.0:1766099561.920969:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1707
00000100:00100000:2.0:1766099561.920977:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432935552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099561.921102:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432935552:12345-0@lo:400:kworker/u8:2.0 Request processed in 125us (2779us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099561.921110:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1707
00000100:00100000:3.0:1766099561.921635:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432935552:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099562.433679:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099563.454162:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099564.477722:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099565.502714:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099566.367601:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099566.367829:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223424
00000100:00100000:1.0:1766099566.370879:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1561
00000100:00100000:1.0:1766099566.370896:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000585c6915 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417223424:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099566.371124:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099566.371335:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000585c6915 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417223424:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 440us (3743us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099566.371348:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1561
00000100:00100000:1.0:1766099566.371452:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099566.372836:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099566.372906:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223360
00000100:00100000:1.0:1766099566.372984:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2519
00000100:00100000:1.0:1766099566.372993:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1de4d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417223360:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099566.373220:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1de4d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417223360:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 227us (391us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099566.373232:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2519
00000100:00100000:1.0:1766099566.373634:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223296
00000100:00100000:1.0:1766099566.373675:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1708
00000100:00100000:1.0:1766099566.373682:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417223296:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099566.373793:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417223296:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 40us (2273us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099566.373802:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1708
00010000:00010000:0.0:1766099566.527685:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099567.037861:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099567.037872:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099567.037876:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099567.037887:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4070
00000100:00100000:1.0:1766099567.037888:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432935936:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099567.037904:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099567.037913:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099567.037942:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000ae936e0 x1851885432936000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099567.037996:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099567.038012:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432936064:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099567.038050:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099567.038084:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000ae936e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936000:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099567.038091:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935936
00000100:00100000:0.0:1766099567.038117:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099567.038149:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936064
00000100:00100000:0.0:1766099567.038158:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2520
00000100:00100000:0.0:1766099567.038167:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432935936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099567.038177:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2521
00000100:00100000:1.0:1766099567.038182:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe958a22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432936064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099567.038239:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432935936:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (246us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099567.038246:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe958a22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432936064:12345-0@lo:400:kworker/u8:2.0 Request processed in 62us (196us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099567.038252:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2520
00000100:00100000:1.0:1766099567.038260:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2521
00000100:00100000:1.0:1766099567.038299:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432936128:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099567.038321:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432936064:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099567.038329:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099567.038369:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432935936:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099567.038369:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936000
00000100:00100000:1.0:1766099567.038392:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936128
00000100:00100000:0.0:1766099567.038402:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1562
00000100:00100000:0.0:1766099567.038407:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef33bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432936000:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099567.038418:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1118
00000100:00100000:1.0:1766099567.038424:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432936128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099567.038456:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432936128:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (131us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099567.038463:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1118
00010000:00080000:1.0:1766099567.038513:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e96bbcf1 x1851885432936128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099574 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099567.038523:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432936128:0@lo:400:kworker/u8:2.0
00010000:02020000:0.0:1766099567.038535:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099567.038573:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef33bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432936000:12345-0@lo:8:kworker/u8:2.0 Request processed in 165us (456us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099567.038581:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1562
00000100:00080000:1.0:1766099567.038592:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099567.038597:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099567.038603:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000ae936e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936000:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099567.038740:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432935872:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099567.038787:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099567.040306:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432935872
00000100:00100000:0.0:1766099567.040375:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1709
00000100:00100000:0.0:1766099567.040381:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432935872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099567.040460:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432935872:12345-0@lo:400:kworker/u8:2.0 Request processed in 78us (1672us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099567.040467:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1709
00000100:00100000:3.0:1766099567.042036:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432935872:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099567.552889:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099568.580238:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099569.599397:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099570.621664:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099571.484985:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099571.485046:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099571.485082:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223488
00000100:00100000:3.0:1766099571.485140:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223552
00000100:00100000:0.0:1766099571.485174:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1710
00000100:00100000:0.0:1766099571.485182:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417223488:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099571.485183:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2522
00000100:00100000:3.0:1766099571.485187:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000faff0b8a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417223552:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099571.485687:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417223488:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 502us (712us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099571.485700:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1710
00000100:00100000:3.0:1766099571.485811:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000faff0b8a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417223552:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 621us (765us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099571.485819:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2522
00000100:00100000:2.0:1766099571.486773:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099571.486881:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223616
00000100:00100000:0.0:1766099571.486939:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1563
00000100:00100000:0.0:1766099571.486946:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417223616:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099571.487045:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099571.491894:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fda231b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417223616:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 4934us (5112us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099571.491918:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1563
00010000:00010000:0.0:1766099571.646959:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099572.164059:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099572.164069:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099572.164073:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099572.164083:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4075
00000100:00080000:0.0:1766099572.164087:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099572.164094:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099572.164124:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c402e8c6 x1851885432936320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099572.164185:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000ae936e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432936384:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164271:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099572.164315:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099572.164321:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432936256:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164344:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099572.164370:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936320:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164425:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099572.164468:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936384
00000100:00100000:0.0:1766099572.164541:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936256
00000100:00100000:0.0:1766099572.164576:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2523
00000100:00100000:0.0:1766099572.164582:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c572263a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432936384:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164636:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c572263a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432936384:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (366us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099572.164645:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2523
00000100:00100000:0.0:1766099572.164665:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2524
00000100:00100000:0.0:1766099572.164668:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000f47354 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432936256:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164690:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000f47354 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432936256:12345-0@lo:400:kworker/u8:3.0 Request processed in 21us (346us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099572.164696:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2524
00000100:00100000:0.0:1766099572.164745:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cde0997c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432936256:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164755:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000ae936e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432936384:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164786:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936320
00000100:00100000:0.0:1766099572.164808:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1564
00000100:00100000:0.0:1766099572.164812:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c81aaacb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432936320:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099572.164885:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432936192:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766099572.164911:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099572.164936:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099572.164968:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c81aaacb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432936320:12345-0@lo:8:kworker/u8:3.0 Request processed in 153us (541us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099572.164968:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099572.164976:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432936448:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.164979:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1564
00000100:00100000:3.0:1766099572.165010:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766099572.165014:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099572.165019:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099572.165023:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936320:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099572.165042:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936448
00000100:00100000:0.0:1766099572.165053:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936192
00000100:00100000:3.0:1766099572.165073:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1119
00000100:00100000:3.0:1766099572.165079:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9fe64a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432936448:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099572.165085:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1711
00000100:00100000:0.0:1766099572.165093:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432936192:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099572.165129:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9fe64a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432936448:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (118us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099572.165135:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432936192:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (199us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099572.165139:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1119
00000100:00100000:0.0:1766099572.165147:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1711
00010000:00080000:3.0:1766099572.165174:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e96bbcf1 x1851885432936448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099579 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099572.165184:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432936448:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099572.165194:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432936192:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099572.678233:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099573.694806:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099574.717747:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099575.742071:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099576.606947:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099576.607009:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099576.607048:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099576.607145:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223744
00000100:00100000:1.0:1766099576.607165:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223680
00000100:00100000:3.0:1766099576.607223:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2525
00000100:00100000:1.0:1766099576.607224:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1712
00000100:00100000:3.0:1766099576.607239:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6ab8e56 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417223744:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099576.607240:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417223680:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099576.607403:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417223680:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 170us (394us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099576.607412:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1712
00000100:00100000:0.0:1766099576.607726:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223808
00000100:00100000:0.0:1766099576.607759:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1565
00000100:00100000:0.0:1766099576.607764:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e4d8abb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417223808:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099576.607861:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099576.609546:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6ab8e56 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417223744:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2304us (2489us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099576.609565:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2525
00000100:00100000:0.0:1766099576.609596:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e4d8abb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417223808:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1828us (2658us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099576.609608:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1565
00010000:00010000:2.0:1766099576.766884:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099577.278392:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099577.278403:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099577.278408:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099577.278418:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4080
00000100:00080000:3.0:1766099577.278422:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099577.278429:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099577.278455:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003dd022ac x1851885432936640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099577.278545:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077d8956b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432936512:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099577.278636:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099577.278743:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432936704:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.279463:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000199cfba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432936768:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099577.279811:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432936576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099577.280065:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099577.280116:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099577.280150:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099577.280201:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936640
00000100:00100000:0.0:1766099577.280246:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1566
00000100:00100000:0.0:1766099577.280253:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000556acc17 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432936640:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099577.280360:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099577.280404:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000556acc17 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432936640:12345-0@lo:8:kworker/u8:2.0 Request processed in 151us (254us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099577.280411:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1566
00000100:00080000:0.0:1766099577.280584:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099577.280588:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099577.280592:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099577.280631:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936512
00000100:00100000:0.0:1766099577.283239:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1713
00000100:00100000:0.0:1766099577.283257:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432936512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.283276:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099577.283365:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936768
00000100:00100000:0.0:1766099577.283392:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432936512:12345-0@lo:400:kworker/u8:2.0 Request processed in 137us (4757us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099577.283402:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1713
00000100:00100000:1.0:1766099577.283412:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1120
00000100:00100000:1.0:1766099577.283418:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abcb39cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432936768:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.283468:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abcb39cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432936768:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (195us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099577.283491:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1120
00010000:00080000:1.0:1766099577.283535:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000199cfba3 x1851885432936768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099584 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099577.283550:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000199cfba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432936768:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.283589:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936704
00000100:00100000:1.0:1766099577.283626:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2526
00000100:00100000:1.0:1766099577.283630:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fb1deee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432936704:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.283746:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fb1deee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432936704:12345-0@lo:400:kworker/u8:2.0 Request processed in 114us (3681us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099577.283754:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2526
00000100:00100000:1.0:1766099577.283789:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432936704:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099577.283935:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099577.284041:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077d8956b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432936512:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.284156:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936576
00000100:00100000:1.0:1766099577.284181:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2527
00000100:00100000:1.0:1766099577.284186:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca5e8fcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432936576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099577.284749:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca5e8fcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432936576:12345-0@lo:400:kworker/u8:2.0 Request processed in 560us (812us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099577.284758:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2527
00000100:00100000:2.0:1766099577.284837:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432936576:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766099577.790908:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099578.813904:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099579.838580:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099580.864056:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099581.724769:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099581.724893:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099581.725000:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099581.725089:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224000
00000100:00100000:1.0:1766099581.725141:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1567
00000100:00100000:1.0:1766099581.725148:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417224000:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099581.725254:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099581.725328:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417224000:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 178us (328us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099581.725336:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1567
00000100:00100000:2.0:1766099581.725451:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223872
00000100:00100000:2.0:1766099581.725525:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1714
00000100:00100000:2.0:1766099581.725530:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417223872:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099581.725568:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417223872:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 38us (676us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099581.725575:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1714
00000100:00100000:3.0:1766099581.726776:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417223936
00000100:00100000:3.0:1766099581.726843:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2528
00000100:00100000:3.0:1766099581.726857:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034f8339a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417223936:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099581.726943:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034f8339a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417223936:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (2187us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099581.726951:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2528
00010000:00010000:0.0:1766099581.885922:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099582.399470:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099582.399496:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099582.399501:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099582.399511:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4085
00000100:00080000:2.0:1766099582.399515:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099582.399521:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099582.399522:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432936832:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099582.399546:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005fda231b x1851885432936960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099582.399593:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ebe6253 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432936896:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099582.399614:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099582.399625:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099582.399645:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099582.399645:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099582.399650:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432937024:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099582.399651:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432937088:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099582.399671:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099582.399673:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099582.399706:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fda231b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936960:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099582.399709:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936896
00000100:00100000:0.0:1766099582.399731:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099582.399758:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937024
00000100:00100000:0.0:1766099582.399762:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2529
00000100:00100000:0.0:1766099582.399769:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432936896:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099582.399775:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2530
00000100:00100000:2.0:1766099582.399778:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432937024:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099582.399821:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432937024:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (149us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099582.399829:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2530
00000100:00100000:1.0:1766099582.399985:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937088
00000100:00100000:1.0:1766099582.400015:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936960
00000100:00100000:1.0:1766099582.400028:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1121
00000100:00100000:1.0:1766099582.400032:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c81aaacb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432937088:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099582.400065:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c81aaacb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432937088:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (392us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099582.400071:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1121
00000100:00100000:1.0:1766099582.400081:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1568
00000100:00100000:1.0:1766099582.400084:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009453e80d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432936960:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099582.400180:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099582.400207:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009453e80d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432936960:12345-0@lo:8:kworker/u8:2.0 Request processed in 122us (476us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099582.400212:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1568
00000100:00100000:1.0:1766099582.400237:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432936832
00000100:00100000:1.0:1766099582.400258:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1715
00000100:00100000:1.0:1766099582.400261:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432936832:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099582.400283:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432936832:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (670us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099582.400288:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1715
00000100:00080000:1.0:1766099582.400323:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099582.400326:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099582.400330:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fda231b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432936960:0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766099582.400364:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000039dc5372 x1851885432937088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099589 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099582.400374:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039dc5372 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432937088:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099582.400382:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce53f395 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432936832:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099582.401377:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432936896:12345-0@lo:400:kworker/u8:2.0 Request processed in 1604us (1747us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099582.401384:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e4d8abb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432937024:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099582.401388:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2529
00000100:00100000:2.0:1766099582.401402:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001ebe6253 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432936896:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099582.912773:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099583.937028:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099584.958430:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099585.981666:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099586.847729:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099586.847793:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099586.847924:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099586.847978:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224128
00000100:00100000:1.0:1766099586.848048:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2531
00000100:00100000:1.0:1766099586.848056:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417224128:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099586.848147:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417224128:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 91us (224us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099586.848155:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2531
00000100:00100000:0.0:1766099586.848330:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224064
00000100:00100000:0.0:1766099586.848370:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1716
00000100:00100000:0.0:1766099586.848375:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417224064:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099586.848405:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417224064:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 29us (612us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099586.848411:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1716
00000100:00100000:2.0:1766099586.848511:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224192
00000100:00100000:2.0:1766099586.848546:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1569
00000100:00100000:2.0:1766099586.848551:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417224192:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099586.850381:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099586.850522:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417224192:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1967us (2806us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099586.850539:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1569
00010000:00010000:0.0:1766099587.005957:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099587.517738:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099587.517748:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099587.517753:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099587.517764:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4090
00000100:00100000:1.0:1766099587.517766:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6ad3f27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432937152:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099587.517770:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099587.517780:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099587.517822:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000020b6667c x1851885432937280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099587.517879:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099587.517913:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432937344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099587.517915:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099587.517924:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432937408:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099587.517967:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099587.517977:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099587.517995:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766099587.518003:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000848a5086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432937216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099587.518014:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432937280:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099587.518034:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099587.518050:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099587.518086:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937152
00000100:00100000:2.0:1766099587.518086:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937408
00000100:00100000:2.0:1766099587.518128:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937280
00000100:00100000:1.0:1766099587.518143:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1717
00000100:00100000:2.0:1766099587.518146:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1122
00000100:00100000:1.0:1766099587.518150:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432937152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099587.518150:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432937408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099587.518197:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432937152:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (320us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099587.518197:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432937408:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (221us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099587.518205:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1717
00000100:00100000:2.0:1766099587.518205:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1122
00000100:00100000:2.0:1766099587.518220:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1570
00000100:00100000:2.0:1766099587.518223:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e67a7300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432937280:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099587.518236:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937344
00000100:00100000:1.0:1766099587.518258:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937216
00000100:00100000:1.0:1766099587.518273:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2532
00000100:00100000:1.0:1766099587.518276:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000056d225d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432937344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099587.518306:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000056d225d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432937344:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (340us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099587.518312:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2532
00010000:02020000:2.0:1766099587.518323:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099587.518351:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e67a7300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432937280:12345-0@lo:8:kworker/u8:3.0 Request processed in 127us (303us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099587.518357:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1570
00000100:00100000:2.0:1766099587.518397:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2533
00000100:00100000:2.0:1766099587.518400:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000199cfba3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432937216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099587.518424:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000199cfba3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432937216:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (393us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099587.518429:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2533
00000100:00080000:2.0:1766099587.519567:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099587.519576:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099587.519581:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432937280:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099587.519615:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000848a5086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432937216:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099587.519625:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432937344:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766099587.519655:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003b84bdab x1851885432937408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099594 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099587.519664:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432937408:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099587.519672:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6ad3f27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432937152:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099588.030566:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099589.060868:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099590.080247:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099591.102050:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099591.965298:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099591.965668:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099591.965726:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224320
00000100:00100000:2.0:1766099591.965812:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2534
00000100:00100000:2.0:1766099591.965820:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3830d7e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417224320:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099591.966236:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224256
00000100:00100000:3.0:1766099591.966268:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1718
00000100:00100000:3.0:1766099591.966273:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417224256:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099591.966876:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3830d7e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417224320:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1049us (1202us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099591.966876:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417224256:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 596us (1580us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099591.966896:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1718
00000100:00100000:2.0:1766099591.966897:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2534
00000100:00100000:0.0:1766099591.976074:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099591.976374:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224384
00000100:00100000:3.0:1766099591.976423:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1571
00000100:00100000:3.0:1766099591.976429:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b36cfcdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417224384:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099591.976550:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099591.976619:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b36cfcdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417224384:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 190us (554us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099591.976627:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1571
00010000:00010000:1.0:1766099592.131687:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099592.639010:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6ad3f27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432937472:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099592.639020:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099592.639026:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099592.639030:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099592.639036:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4095
00000100:00080000:3.0:1766099592.639039:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099592.639046:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099592.639071:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bbe45651 x1851885432937600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099592.639116:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099592.639117:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432937728:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099592.639153:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099592.639158:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432937536:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099592.639186:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099592.639215:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000848a5086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432937664:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099592.639228:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432937600:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099592.639240:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099592.639249:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099592.639287:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937472
00000100:00100000:3.0:1766099592.639306:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937536
00000100:00100000:3.0:1766099592.639332:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937664
00000100:00100000:2.0:1766099592.639335:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1719
00000100:00100000:2.0:1766099592.639341:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432937472:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099592.639348:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2535
00000100:00100000:3.0:1766099592.639351:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018198959 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432937536:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099592.639390:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432937472:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (275us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099592.639390:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018198959 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432937536:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (203us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099592.639398:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2535
00000100:00100000:3.0:1766099592.639415:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2536
00000100:00100000:3.0:1766099592.639418:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000663ddca5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432937664:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099592.639447:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000663ddca5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432937664:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (206us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099592.639453:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2536
00000100:00100000:3.0:1766099592.639509:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000848a5086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432937664:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099592.639552:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1719
00000100:00100000:3.0:1766099592.639556:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937728
00000100:00100000:3.0:1766099592.639579:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937600
00000100:00100000:2.0:1766099592.639584:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1123
00000100:00100000:2.0:1766099592.639588:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432937728:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099592.639622:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432937728:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (469us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099592.643799:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1123
00000100:00100000:3.0:1766099592.643818:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1572
00000100:00100000:3.0:1766099592.643827:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000612198b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432937600:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099592.643895:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b84bdab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432937536:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766099592.643928:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:2.0:1766099592.643939:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000020b6667c x1851885432937728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099599 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099592.643954:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432937728:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099592.643983:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6ad3f27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432937472:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099592.644025:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000612198b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432937600:12345-0@lo:8:kworker/u8:3.0 Request processed in 198us (4776us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099592.644032:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1572
00000100:00080000:3.0:1766099592.644064:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099592.644069:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099592.644074:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432937600:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766099593.152099:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099594.174137:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099595.202314:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099596.222916:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099597.092075:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099597.092180:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099597.092505:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224512
00000100:00100000:2.0:1766099597.092582:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2537
00000100:00100000:2.0:1766099597.092590:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095212d78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417224512:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099597.092753:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095212d78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417224512:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 162us (573us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099597.092761:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2537
00000100:00100000:1.0:1766099597.094788:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224448
00000100:00100000:0.0:1766099597.095010:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099597.095108:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224576
00000100:00100000:3.0:1766099597.095151:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1573
00000100:00100000:3.0:1766099597.095158:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a71a08db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417224576:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099597.095577:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099597.095660:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a71a08db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417224576:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 501us (655us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099597.095670:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1573
00000100:00100000:1.0:1766099597.095851:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1720
00000100:00100000:1.0:1766099597.095859:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417224448:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099597.095901:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417224448:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 43us (3838us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099597.095907:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1720
00010000:00010000:1.0:1766099597.266886:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099597.759431:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099597.759440:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099597.759445:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099597.759456:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4101
00000100:00080000:1.0:1766099597.759460:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099597.759466:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099597.759524:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000019993a78 x1851885432937920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099597.759586:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000066267589 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432938048:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099597.759725:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099597.761906:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c45bca0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432937792:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099597.761988:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099597.762061:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432937984:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099597.762891:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000019993a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432937920:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099597.762944:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099597.763002:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938048
00000100:00100000:1.0:1766099597.763053:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937920
00000100:00100000:1.0:1766099597.763075:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124
00000100:00100000:1.0:1766099597.763082:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432938048:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099597.763146:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432938048:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (3423us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099597.763155:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124
00000100:00100000:1.0:1766099597.763174:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1574
00000100:00100000:1.0:1766099597.763177:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432937920:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099597.763280:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099597.763312:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432937920:12345-0@lo:8:kworker/u8:2.0 Request processed in 133us (368us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099597.763318:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1574
00000100:00080000:1.0:1766099597.763367:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099597.763372:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099597.763378:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000019993a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432937920:0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766099597.763412:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000066267589 x1851885432938048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099604 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099597.763424:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000066267589 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432938048:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099597.763510:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b735f288 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432937856:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099597.763934:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937792
00000100:00100000:3.0:1766099597.763968:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1721
00000100:00100000:3.0:1766099597.763972:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432937792:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099597.764024:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432937792:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (2036us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099597.764030:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1721
00000100:00100000:3.0:1766099597.764106:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c45bca0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432937792:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099597.769578:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099597.769770:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937984
00000100:00100000:3.0:1766099597.769844:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2538
00000100:00100000:3.0:1766099597.769852:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9bbfa03 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432937984:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099597.769969:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9bbfa03 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432937984:12345-0@lo:400:kworker/u8:2.0 Request processed in 116us (397us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099597.769978:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2538
00000100:00100000:3.0:1766099597.770078:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432937984:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099597.771048:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099597.771588:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432937856
00000100:00100000:3.0:1766099597.771646:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2539
00000100:00100000:3.0:1766099597.771653:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066267589 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432937856:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099597.771751:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066267589 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432937856:12345-0@lo:400:kworker/u8:2.0 Request processed in 98us (708us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099597.771759:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2539
00000100:00100000:1.0:1766099597.772129:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b735f288 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432937856:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099598.270284:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099599.294572:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099600.319084:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099601.341647:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099602.207006:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099602.207099:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099602.207588:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224640
00000100:00100000:1.0:1766099602.207668:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1722
00000100:00100000:1.0:1766099602.207675:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417224640:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099602.207780:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417224640:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 104us (782us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099602.207789:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1722
00000100:00100000:2.0:1766099602.207893:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099602.208234:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224768
00000100:00100000:1.0:1766099602.208269:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1575
00000100:00100000:1.0:1766099602.208274:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a81cfa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417224768:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099602.208362:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099602.208403:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a81cfa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417224768:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 128us (511us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099602.208410:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1575
00000100:00100000:3.0:1766099602.208754:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224704
00000100:00100000:3.0:1766099602.208797:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2540
00000100:00100000:3.0:1766099602.208801:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c574a8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417224704:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099602.208852:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c574a8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417224704:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 49us (1754us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099602.208859:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2540
00010000:00010000:2.0:1766099602.366495:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099602.877802:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099602.878579:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b735f288 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432938112:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099602.878667:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099602.878787:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938112
00000100:00100000:2.0:1766099602.878839:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1723
00000100:00100000:2.0:1766099602.878846:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432938112:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099602.878902:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432938112:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (234us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099602.878911:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1723
00000100:00100000:2.0:1766099602.878976:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b735f288 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432938112:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099602.880260:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099602.880267:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099602.880271:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099602.880277:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4106
00000100:00080000:1.0:1766099602.880282:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099602.880289:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099602.880315:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d4f110b0 x1851885432938240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099602.880371:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d4f110b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432938240:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099602.880421:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099602.880466:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938240
00000100:00100000:1.0:1766099602.880521:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1576
00000100:00100000:1.0:1766099602.880526:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432938240:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099602.880639:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099602.880676:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432938240:12345-0@lo:8:kworker/u8:2.0 Request processed in 150us (256us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099602.880683:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1576
00000100:00080000:1.0:1766099602.880730:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099602.880734:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099602.880738:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d4f110b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432938240:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099602.880759:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099602.880767:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1de4d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432938368:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.880792:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099602.880815:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938368
00000100:00100000:1.0:1766099602.880841:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1125
00000100:00100000:1.0:1766099602.880845:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432938368:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.880880:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432938368:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (90us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099602.880887:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1125
00010000:00080000:1.0:1766099602.880911:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e1de4d22 x1851885432938368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099609 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099602.880922:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1de4d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432938368:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.880957:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090ca65cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432938304:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.880986:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099602.881003:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099602.881008:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000019993a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432938176:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.881026:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099602.881051:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938304
00000100:00100000:1.0:1766099602.881076:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938176
00000100:00100000:1.0:1766099602.881091:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2541
00000100:00100000:1.0:1766099602.881095:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1de4d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432938304:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.881123:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1de4d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432938304:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (138us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099602.881130:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2541
00000100:00100000:1.0:1766099602.881138:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2542
00000100:00100000:1.0:1766099602.881142:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000faff0b8a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432938176:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.881160:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000faff0b8a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432938176:12345-0@lo:400:kworker/u8:2.0 Request processed in 18us (134us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099602.881165:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2542
00000100:00100000:1.0:1766099602.881191:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000019993a78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432938176:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099602.881201:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090ca65cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432938304:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099603.390066:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099604.415805:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099605.437829:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099606.462566:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099607.332468:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099607.332552:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099607.332574:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099607.332705:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224896
00000100:00100000:3.0:1766099607.332761:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2543
00000100:00100000:3.0:1766099607.332768:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019993a78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417224896:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099607.332836:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019993a78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417224896:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 67us (262us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099607.332844:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2543
00000100:00100000:0.0:1766099607.332918:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224960
00000100:00100000:0.0:1766099607.332951:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1577
00000100:00100000:0.0:1766099607.332955:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090ca65cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417224960:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099607.333055:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099607.333082:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090ca65cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417224960:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 127us (623us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099607.333089:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1577
00000100:00100000:2.0:1766099607.333322:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417224832
00000100:00100000:2.0:1766099607.333358:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1724
00000100:00100000:2.0:1766099607.333362:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417224832:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099607.333399:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417224832:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 36us (848us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099607.333406:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1724
00010000:00010000:1.0:1766099607.485728:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099607.998309:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099607.998318:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099607.998322:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099607.998334:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4111
00000100:00080000:1.0:1766099607.998339:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099607.998347:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099607.998389:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000016c31705 x1851885432938560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099607.998448:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa17f27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432938624:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099607.998566:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099607.998611:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099607.998617:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ed1b667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432938496:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099607.998668:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099607.998696:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000016c31705 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432938560:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099607.998726:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099607.998772:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938624
00000100:00100000:1.0:1766099607.998837:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938496
00000100:00100000:1.0:1766099607.998860:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2544
00000100:00100000:1.0:1766099607.998866:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d0dc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432938624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.998995:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432938688:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999027:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099607.999049:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099607.999053:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034f8339a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432938432:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999091:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099607.999120:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938432
00000100:00100000:0.0:1766099607.999147:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1725
00000100:00100000:0.0:1766099607.999151:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432938432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999187:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432938432:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (95us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099607.999194:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1725
00000100:00100000:0.0:1766099607.999246:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034f8339a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432938432:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999271:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938560
00000100:00100000:0.0:1766099607.999291:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938688
00000100:00100000:0.0:1766099607.999311:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1126
00000100:00100000:0.0:1766099607.999315:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8fb2439 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432938688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999349:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8fb2439 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432938688:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (322us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099607.999360:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1126
00000100:00100000:0.0:1766099607.999383:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1578
00000100:00100000:0.0:1766099607.999386:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432938560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099607.999525:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099607.999561:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432938560:12345-0@lo:8:kworker/u8:3.0 Request processed in 173us (835us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099607.999567:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1578
00000100:00080000:0.0:1766099607.999598:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099607.999602:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099607.999605:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000016c31705 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432938560:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766099607.999628:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000be044d66 x1851885432938688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099614 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099607.999638:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432938688:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999682:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2545
00000100:00100000:0.0:1766099607.999688:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f13f01a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432938496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099607.999725:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f13f01a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432938496:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (1058us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099607.999731:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2545
00000100:00100000:0.0:1766099607.999764:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ed1b667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432938496:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099608.001723:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d0dc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432938624:12345-0@lo:400:kworker/u8:3.0 Request processed in 2846us (3149us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099608.001743:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2544
00000100:00100000:2.0:1766099608.003593:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa17f27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432938624:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099608.509693:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099609.536754:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099610.557667:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099611.581805:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099612.445074:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099612.446498:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225024
00000100:00100000:0.0:1766099612.447219:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1726
00000100:00100000:0.0:1766099612.447230:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417225024:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099612.447354:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417225024:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 124us (2294us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099612.447365:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1726
00000100:00100000:1.0:1766099612.448948:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099612.450348:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225088
00000100:00100000:0.0:1766099612.450465:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2546
00000100:00100000:0.0:1766099612.450473:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eaa17f27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417225088:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099612.450738:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eaa17f27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417225088:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 264us (1799us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099612.450747:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2546
00000100:00100000:1.0:1766099612.452398:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099612.452505:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225152
00000100:00100000:3.0:1766099612.452557:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1579
00000100:00100000:3.0:1766099612.452565:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417225152:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099612.452806:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099612.452883:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417225152:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 318us (494us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099612.452893:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1579
00010000:00010000:1.0:1766099612.606795:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099613.117870:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099613.117880:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:3.0:1766099613.117883:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766099613.117885:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099613.117897:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4116
00000100:00080000:2.0:1766099613.117901:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099613.117909:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099613.117912:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432938752:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099613.117938:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002a6cc55b x1851885432938880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099613.118000:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a7ccce10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432938816:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099613.118023:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099613.118058:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099613.118069:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008358d193 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432939008:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099613.118092:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099613.118101:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432938944:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099613.118106:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099613.118135:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099613.118200:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938752
00000100:00100000:1.0:1766099613.118234:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939008
00000100:00100000:3.0:1766099613.118258:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1727
00000100:00100000:3.0:1766099613.118267:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432938752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099613.118276:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1127
00000100:00100000:1.0:1766099613.118281:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432939008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099613.118337:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432938752:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (316us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099613.118341:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432939008:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (236us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099613.118353:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1727
00000100:00100000:1.0:1766099613.118354:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1127
00000100:00100000:2.0:1766099613.118437:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938816
00000100:00100000:2.0:1766099613.118501:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938944
00000100:00100000:2.0:1766099613.118524:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2547
00000100:00100000:2.0:1766099613.118529:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047f8d5e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432938816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099613.118571:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047f8d5e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432938816:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (516us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099613.118578:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2547
00000100:00100000:2.0:1766099613.118591:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2548
00000100:00100000:2.0:1766099613.118594:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a8e1966 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432938944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099613.118628:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a8e1966 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432938944:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (486us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099613.118635:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2548
00000100:00100000:2.0:1766099613.118675:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432938944:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099613.118712:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a6cc55b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432938880:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099613.118745:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099613.119012:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a7ccce10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432938816:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099613.119171:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432938880
00000100:00100000:1.0:1766099613.119195:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1580
00000100:00100000:1.0:1766099613.119199:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003204340c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432938880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099613.119293:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099613.119334:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003204340c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432938880:12345-0@lo:8:kworker/u8:2.0 Request processed in 134us (588us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099613.119341:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1580
00010000:00080000:3.0:1766099613.119358:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008358d193 x1851885432939008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099620 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099613.119372:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008358d193 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432939008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099613.119388:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432938752:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099613.119562:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099613.119567:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099613.119572:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a6cc55b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432938880:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766099613.629794:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099614.658303:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099615.685065:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099616.701776:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099617.570169:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099617.570225:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099617.570319:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225216
00000100:00100000:1.0:1766099617.570384:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1728
00000100:00100000:1.0:1766099617.570392:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417225216:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099617.570472:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417225216:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 81us (315us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099617.570516:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1728
00000100:00100000:3.0:1766099617.570755:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225280
00000100:00100000:3.0:1766099617.570802:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2549
00000100:00100000:3.0:1766099617.570806:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7ccce10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417225280:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099617.570847:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7ccce10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417225280:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 39us (622us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099617.570854:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2549
00000100:00100000:2.0:1766099617.586417:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099617.586533:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225344
00000100:00100000:0.0:1766099617.586604:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1581
00000100:00100000:0.0:1766099617.586612:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003837ea46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417225344:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099617.586717:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099617.587503:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003837ea46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417225344:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 858us (1067us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099617.587515:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1581
00010000:00010000:2.0:1766099617.725747:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099618.237910:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099618.237922:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099618.237926:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099618.237937:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4121
00000100:00080000:2.0:1766099618.237941:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099618.237948:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099618.237976:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000cbb82e8f x1851885432939200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099618.238075:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432939136:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099618.238157:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099618.238195:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099618.238201:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005834876b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432939264:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099618.238221:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099618.238516:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939136
00000100:00100000:3.0:1766099618.238564:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939264
00000100:00100000:3.0:1766099618.238599:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2550
00000100:00100000:3.0:1766099618.238605:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c68b1dd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432939136:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099618.238666:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c68b1dd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432939136:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (510us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099618.238675:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2550
00000100:00100000:3.0:1766099618.238693:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2551
00000100:00100000:3.0:1766099618.238697:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076cc20a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432939264:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099618.238725:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076cc20a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432939264:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (503us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099618.238731:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2551
00000100:00100000:0.0:1766099618.238739:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a889833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432939328:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099618.238767:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005834876b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432939264:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099618.238774:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099618.238778:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432939136:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099618.238791:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099618.238796:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432939072:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099618.238822:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099618.238957:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbb82e8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432939200:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099618.238988:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099618.239023:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939072
00000100:00100000:1.0:1766099618.239030:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939328
00000100:00100000:3.0:1766099618.239050:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1729
00000100:00100000:1.0:1766099618.239051:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939200
00000100:00100000:3.0:1766099618.239054:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432939072:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099618.239067:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128
00000100:00100000:1.0:1766099618.239070:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432939328:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099618.239086:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432939072:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (265us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099618.239093:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1729
00000100:00100000:1.0:1766099618.239095:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432939328:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (322us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099618.239100:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128
00000100:00100000:1.0:1766099618.239110:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1582
00000100:00100000:1.0:1766099618.239113:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be044d66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432939200:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099618.239122:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432939072:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766099618.239130:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005a889833 x1851885432939328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099625 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099618.239140:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a889833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432939328:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099618.239215:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099618.239266:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be044d66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432939200:12345-0@lo:8:kworker/u8:2.0 Request processed in 151us (277us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099618.239276:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1582
00000100:00080000:3.0:1766099618.239290:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099618.239294:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099618.239297:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbb82e8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432939200:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766099618.752563:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099619.775869:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099620.801814:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099621.821942:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099622.684865:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099622.684925:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099622.685188:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225472
00000100:00100000:3.0:1766099622.685253:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2552
00000100:00100000:3.0:1766099622.685260:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c5369a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417225472:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099622.685352:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c5369a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417225472:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 91us (499us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099622.685361:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2552
00000100:00100000:3.0:1766099622.685387:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225408
00000100:00100000:3.0:1766099622.685417:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1730
00000100:00100000:3.0:1766099622.685420:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417225408:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099622.685447:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417225408:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 25us (522us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099622.685454:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1730
00000100:00100000:2.0:1766099622.687387:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099622.687517:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225536
00000100:00100000:0.0:1766099622.687570:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1583
00000100:00100000:0.0:1766099622.687577:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417225536:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099622.687678:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099622.687756:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417225536:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 178us (376us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099622.687764:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1583
00010000:00010000:1.0:1766099622.845743:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099623.357788:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a7d46da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432939392:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766099623.357791:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099623.357809:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099623.357816:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099623.357825:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4126
00000100:00080000:2.0:1766099623.357829:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099623.357837:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099623.357864:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432939456:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099623.357870:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005f11e586 x1851885432939520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099623.357890:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099623.357901:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099623.357930:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099623.357941:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432939584:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099623.357963:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099623.357970:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099623.357971:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000388d2359 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432939648:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099623.357986:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005f11e586 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432939520:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099623.358001:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099623.358011:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939456
00000100:00100000:0.0:1766099623.358051:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939584
00000100:00100000:2.0:1766099623.358056:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099623.358076:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2553
00000100:00100000:0.0:1766099623.358084:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000235f24de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432939456:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099623.358149:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000235f24de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432939456:12345-0@lo:400:kworker/u8:3.0 Request processed in 66us (248us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099623.358154:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2554
00000100:00100000:2.0:1766099623.358169:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f01f3690 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432939584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099623.358170:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2553
00000100:00100000:2.0:1766099623.358241:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f01f3690 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432939584:12345-0@lo:400:kworker/u8:3.0 Request processed in 74us (269us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099623.358251:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2554
00000100:00100000:0.0:1766099623.358262:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023de6956 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432939584:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099623.358279:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026f8c99c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432939456:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099623.358318:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939648
00000100:00100000:0.0:1766099623.358352:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939520
00000100:00100000:0.0:1766099623.358375:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1129
00000100:00100000:0.0:1766099623.358381:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432939648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099623.358427:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432939648:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (426us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099623.358434:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1129
00000100:00100000:0.0:1766099623.358454:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1584
00000100:00100000:0.0:1766099623.358457:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432939520:12345-0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766099623.358471:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000388d2359 x1851885432939648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099630 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099623.358513:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000388d2359 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432939648:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766099623.358598:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099623.358649:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432939520:12345-0@lo:8:kworker/u8:3.0 Request processed in 190us (595us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099623.358658:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1584
00000100:00100000:1.0:1766099623.358663:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939392
00000100:00100000:1.0:1766099623.358715:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1731
00000100:00100000:1.0:1766099623.358721:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432939392:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099623.358796:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432939392:12345-0@lo:400:kworker/u8:3.0 Request processed in 74us (916us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099623.358803:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1731
00000100:00080000:2.0:1766099623.360025:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099623.360034:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099623.360040:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005f11e586 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432939520:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099623.366280:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a7d46da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432939392:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099623.869721:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099624.895090:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099625.917730:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099626.943010:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099627.806958:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099627.806997:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099627.807067:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099627.807712:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225600
00000100:00100000:1.0:1766099627.807769:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1732
00000100:00100000:1.0:1766099627.807775:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417225600:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099627.807843:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417225600:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (895us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099627.807864:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1732
00000100:00100000:3.0:1766099627.808056:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225728
00000100:00100000:3.0:1766099627.808083:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1585
00000100:00100000:3.0:1766099627.808087:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417225728:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099627.808178:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099627.808208:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417225728:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 120us (1141us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099627.808215:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1585
00000100:00100000:0.0:1766099627.808922:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225664
00000100:00100000:0.0:1766099627.808975:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2555
00000100:00100000:0.0:1766099627.808980:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f11e586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417225664:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099627.809043:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f11e586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417225664:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 62us (2046us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099627.809050:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2555
00010000:00010000:1.0:1766099627.966047:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099628.477645:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432939712:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099628.477694:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f78b201d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432939776:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099628.477719:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099628.477738:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099628.477767:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939712
00000100:00100000:2.0:1766099628.477806:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1733
00000100:00100000:2.0:1766099628.477811:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432939712:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099628.477851:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432939712:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (133us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099628.477858:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1733
00000100:00100000:2.0:1766099628.477898:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432939712:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099628.477929:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939776
00000100:00100000:2.0:1766099628.477956:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2556
00000100:00100000:2.0:1766099628.477960:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421476e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432939776:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099628.481112:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421476e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432939776:12345-0@lo:400:kworker/u8:3.0 Request processed in 3143us (3365us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099628.481128:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2556
00000100:00100000:3.0:1766099628.481239:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f78b201d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432939776:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099628.481348:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099628.481354:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099628.481357:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099628.481362:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4131
00000100:00080000:0.0:1766099628.481365:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099628.481370:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099628.481401:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000665bb208 x1851885432939840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099628.481474:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000079b9f738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432939904:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099628.481557:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099628.482343:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939904
00000100:00100000:1.0:1766099628.482385:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2557
00000100:00100000:1.0:1766099628.482392:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eea6599 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432939904:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099628.482521:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432939840:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099628.482556:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099628.482589:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939840
00000100:00100000:2.0:1766099628.482611:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1586
00000100:00100000:2.0:1766099628.482615:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432939840:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099628.482717:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099628.482747:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432939840:12345-0@lo:8:kworker/u8:3.0 Request processed in 131us (191us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099628.482754:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1586
00000100:00080000:2.0:1766099628.482786:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099628.482790:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099628.482793:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432939840:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099628.483576:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099628.483586:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061def75d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432939968:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099628.483620:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099628.483646:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432939968
00000100:00100000:0.0:1766099628.483674:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1130
00000100:00100000:0.0:1766099628.483678:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f373762c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432939968:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099628.483712:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f373762c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432939968:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (92us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099628.483718:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1130
00010000:00080000:0.0:1766099628.483745:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000061def75d x1851885432939968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099635 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099628.483757:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061def75d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432939968:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099628.484378:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eea6599 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432939904:12345-0@lo:400:kworker/u8:3.0 Request processed in 1983us (2819us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099628.484388:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2557
00000100:00100000:3.0:1766099628.484461:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000079b9f738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432939904:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099628.989700:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099630.014091:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099631.037797:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099632.061917:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099632.925766:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099632.955341:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099632.955401:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099632.955545:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225792
00000100:00100000:3.0:1766099632.955604:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1734
00000100:00100000:3.0:1766099632.955611:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417225792:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099632.955682:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417225792:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 70us (348us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099632.955689:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1734
00000100:00100000:1.0:1766099632.956730:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225920
00000100:00100000:0.0:1766099632.957339:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225856
00000100:00100000:0.0:1766099632.957399:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2558
00000100:00100000:0.0:1766099632.957406:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417225856:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099632.957499:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf6d59f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417225856:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 69us (31715us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099632.957507:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2558
00000100:00100000:2.0:1766099632.958049:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1587
00000100:00100000:2.0:1766099632.958055:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a7d46da pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417225920:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099632.958147:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099632.958544:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a7d46da pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417225920:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 487us (3142us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099632.958553:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1587
00010000:00010000:2.0:1766099633.087565:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099633.597782:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099633.597788:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099633.597791:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099633.597802:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4136
00000100:00080000:3.0:1766099633.597806:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099633.597813:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099633.597838:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007e2ba551 x1851885432940160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099633.597855:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071acbd6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432940096:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099633.597893:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000817fb53b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432940224:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099633.597989:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000769cb361 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432940032:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099633.598012:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc6b3d7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432940288:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099633.599100:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099633.599180:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432940160:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099633.599220:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099633.599355:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099633.599443:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940096
00000100:00100000:0.0:1766099633.599532:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2559
00000100:00100000:0.0:1766099633.599538:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026f8c99c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432940096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099633.599607:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026f8c99c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432940096:12345-0@lo:400:kworker/u8:3.0 Request processed in 69us (512us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099633.599616:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2559
00000100:00100000:0.0:1766099633.599669:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071acbd6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432940096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099633.599770:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099633.599829:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940160
00000100:00100000:1.0:1766099633.599871:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940288
00000100:00100000:1.0:1766099633.599911:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1131
00000100:00100000:1.0:1766099633.599919:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016c31705 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432940288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099633.599933:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1588
00000100:00100000:0.0:1766099633.599941:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432940160:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099633.599980:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016c31705 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432940288:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (210us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099633.599990:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1131
00010000:00080000:1.0:1766099633.600032:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fc6b3d7a x1851885432940288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099640 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00010000:02020000:0.0:1766099633.600040:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099633.600057:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc6b3d7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432940288:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099633.600102:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432940160:12345-0@lo:8:kworker/u8:3.0 Request processed in 158us (880us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099633.600113:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1588
00000100:00080000:1.0:1766099633.600122:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099633.600128:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099633.600134:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432940160:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099633.600288:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940032
00000100:00100000:2.0:1766099633.600334:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1735
00000100:00100000:2.0:1766099633.600339:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432940032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099633.601904:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099633.602071:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432940032:12345-0@lo:400:kworker/u8:3.0 Request processed in 1724us (2714us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099633.602085:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1735
00000100:00100000:0.0:1766099633.602255:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000769cb361 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432940032:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099633.605987:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940224
00000100:00100000:1.0:1766099633.606072:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2560
00000100:00100000:1.0:1766099633.606079:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3ebfac1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432940224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099633.606191:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3ebfac1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432940224:12345-0@lo:400:kworker/u8:3.0 Request processed in 111us (4292us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099633.606199:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2560
00000100:00100000:3.0:1766099633.606329:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000817fb53b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432940224:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099634.112828:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099635.138078:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099636.157760:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099637.181911:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099638.047526:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099638.049718:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417225984
00000100:00100000:2.0:1766099638.049804:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1736
00000100:00100000:2.0:1766099638.049813:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417225984:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099638.050173:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099638.051214:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417225984:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1394us (3693us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099638.051231:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1736
00000100:00100000:2.0:1766099638.051307:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226048
00000100:00100000:2.0:1766099638.051354:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2561
00000100:00100000:2.0:1766099638.051360:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000817fb53b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417226048:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099638.052846:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000817fb53b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417226048:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1477us (2666us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099638.052861:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2561
00000100:00100000:0.0:1766099638.053646:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099638.055025:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226112
00000100:00100000:1.0:1766099638.055094:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1589
00000100:00100000:1.0:1766099638.055102:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061def75d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417226112:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099638.055206:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099638.058101:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061def75d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417226112:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 2984us (4445us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099638.058124:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1589
00010000:00010000:2.0:1766099638.206776:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099638.718957:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099638.718966:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099638.718970:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099638.718982:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4142
00000100:00080000:2.0:1766099638.718986:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099638.718994:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099638.719021:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000041477718 x1851885432940480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099638.719079:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432940480:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099638.719164:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099638.719241:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940480
00000100:00100000:2.0:1766099638.719284:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1590
00000100:00100000:2.0:1766099638.719291:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432940480:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099638.719388:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099638.719451:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432940480:12345-0@lo:8:kworker/u8:3.0 Request processed in 160us (288us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099638.719459:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1590
00000100:00080000:2.0:1766099638.719535:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099638.719539:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099638.719544:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432940480:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099638.719633:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a889833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432940416:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099638.719671:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099638.719703:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099638.719709:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432940544:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099638.719731:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099638.719762:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940416
00000100:00100000:0.0:1766099638.719796:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940544
00000100:00100000:0.0:1766099638.719815:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2562
00000100:00100000:0.0:1766099638.719820:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a39919c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432940416:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099638.719863:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a39919c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432940416:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (192us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099638.719869:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2562
00000100:00100000:0.0:1766099638.719886:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2563
00000100:00100000:0.0:1766099638.719889:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042186850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432940544:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099638.719915:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042186850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432940544:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (183us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099638.719921:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2563
00000100:00100000:0.0:1766099638.719964:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432940544:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099638.719973:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a889833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432940416:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099638.720338:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbb82e8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432940352:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099638.720379:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099638.720412:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099638.720417:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005834876b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432940608:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099638.720456:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099638.722658:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940352
00000100:00100000:0.0:1766099638.722730:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1737
00000100:00100000:0.0:1766099638.722738:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432940352:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099638.722841:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432940352:12345-0@lo:400:kworker/u8:3.0 Request processed in 101us (2461us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099638.722850:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1737
00000100:00100000:3.0:1766099638.722873:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbb82e8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432940352:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099638.723192:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940608
00000100:00100000:2.0:1766099638.723237:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1132
00000100:00100000:2.0:1766099638.723242:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074023263 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432940608:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099638.723298:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074023263 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432940608:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (2841us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099638.723304:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1132
00010000:00080000:3.0:1766099638.723730:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005834876b x1851885432940608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099645 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099638.723747:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005834876b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432940608:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099639.231246:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099640.255359:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099641.277702:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099642.301695:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099643.167267:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099643.167313:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099643.167356:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099643.167470:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226304
00000100:00100000:3.0:1766099643.167545:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1591
00000100:00100000:3.0:1766099643.167552:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b735f288 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417226304:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099643.167657:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099643.167714:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b735f288 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417226304:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 162us (359us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099643.167723:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1591
00000100:00100000:2.0:1766099643.167835:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226176
00000100:00100000:2.0:1766099643.167874:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1738
00000100:00100000:2.0:1766099643.167879:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226176:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099643.167913:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226176:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 33us (655us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099643.167920:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1738
00000100:00100000:0.0:1766099643.168037:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226240
00000100:00100000:0.0:1766099643.168076:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2564
00000100:00100000:0.0:1766099643.168081:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034f8339a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417226240:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099643.168111:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034f8339a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417226240:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 30us (799us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099643.168118:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2564
00010000:00010000:1.0:1766099643.325698:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099643.837710:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432940672:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099643.837790:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099643.837883:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940672
00000100:00100000:0.0:1766099643.837953:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1739
00000100:00100000:0.0:1766099643.837960:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432940672:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099643.838053:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432940672:12345-0@lo:400:kworker/u8:2.0 Request processed in 92us (264us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099643.838061:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1739
00000100:00100000:0.0:1766099643.838124:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432940672:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099643.839632:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c45bca0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432940736:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099643.839718:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766099643.839926:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099643.839933:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099643.839938:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099643.839943:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4147
00000100:00080000:1.0:1766099643.839945:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099643.839952:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099643.839982:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f1197d22 x1851885432940800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099643.840036:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099643.840042:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abcb39cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432940864:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099643.840070:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099643.840116:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940864
00000100:00100000:3.0:1766099643.840167:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2565
00000100:00100000:3.0:1766099643.840173:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006301b41a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432940864:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099643.840225:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006301b41a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432940864:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (154us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099643.840232:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2565
00000100:00100000:3.0:1766099643.840279:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abcb39cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432940864:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099643.840317:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432940800:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099643.840346:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099643.840378:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940800
00000100:00100000:3.0:1766099643.840407:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1592
00000100:00100000:3.0:1766099643.840411:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432940800:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099643.840527:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099643.840563:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432940800:12345-0@lo:8:kworker/u8:2.0 Request processed in 150us (217us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099643.840573:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1592
00000100:00100000:2.0:1766099643.840582:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940736
00000100:00080000:3.0:1766099643.840605:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099643.840610:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099643.840615:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432940800:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099643.840658:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2566
00000100:00100000:1.0:1766099643.840765:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099643.840777:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000157c8bb4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432940928:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099643.840804:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000022d6139c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432940736:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099643.840815:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099643.840886:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940928
00000100:00100000:0.0:1766099643.840935:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1133
00000100:00100000:0.0:1766099643.840939:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432940928:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099643.841014:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432940928:12345-0@lo:400:kworker/u8:2.0 Request processed in 73us (204us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099643.841021:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1133
00000100:00100000:2.0:1766099643.841597:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000022d6139c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432940736:12345-0@lo:400:kworker/u8:2.0 Request processed in 797us (1879us total) trans 0 rc 0/0
00010000:00080000:1.0:1766099643.841681:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000157c8bb4 x1851885432940928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099650 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099643.841696:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000157c8bb4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432940928:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099643.841977:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c45bca0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432940736:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099643.842161:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2566
00010000:00010000:1.0:1766099644.350235:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099645.374592:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099646.398568:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099647.422450:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099648.284665:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099648.284812:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226496
00000100:00100000:2.0:1766099648.284859:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1593
00000100:00100000:2.0:1766099648.284865:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417226496:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099648.284958:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099648.285020:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417226496:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 154us (362us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099648.285027:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1593
00000100:00100000:1.0:1766099648.285716:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099648.286212:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099648.286283:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226432
00000100:00100000:3.0:1766099648.286336:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2567
00000100:00100000:3.0:1766099648.286342:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b09a86b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417226432:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099648.286415:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b09a86b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417226432:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 72us (204us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099648.286422:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2567
00000100:00100000:2.0:1766099648.286556:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226368
00000100:00100000:2.0:1766099648.286591:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1740
00000100:00100000:2.0:1766099648.286595:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226368:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099648.286622:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226368:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 26us (908us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099648.286628:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1740
00010000:00010000:2.0:1766099648.446185:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099648.958199:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099648.958837:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007de97d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432940992:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099648.958907:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099648.959268:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e67a7300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432941056:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099648.959300:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099648.960517:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432940992
00000100:00100000:0.0:1766099648.960571:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1741
00000100:00100000:0.0:1766099648.960576:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432940992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099648.960702:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432940992:12345-0@lo:400:kworker/u8:2.0 Request processed in 125us (1796us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099648.960710:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1741
00000100:00100000:2.0:1766099648.963538:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941056
00000100:00100000:2.0:1766099648.963921:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2568
00000100:00100000:2.0:1766099648.963927:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004bd75cc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432941056:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099648.965428:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099648.965439:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099648.965443:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099648.965449:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4152
00000100:00080000:1.0:1766099648.965452:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099648.965459:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099648.965555:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a7d0a8c1 x1851885432941120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099648.965707:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a7d0a8c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432941120:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099648.965761:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099648.965798:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432941184:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099648.965824:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099648.965859:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941120
00000100:00100000:1.0:1766099648.965891:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1594
00000100:00100000:1.0:1766099648.965897:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432941120:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099648.965992:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099648.966027:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432941120:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (266us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099648.966034:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1594
00000100:00080000:1.0:1766099648.966081:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099648.966086:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099648.966089:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a7d0a8c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432941120:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099648.966110:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b04885a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432941248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099648.966133:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099648.966170:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941248
00000100:00100000:1.0:1766099648.966195:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1134
00000100:00100000:1.0:1766099648.966199:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432941248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099648.966228:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432941248:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (96us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099648.966234:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1134
00000100:00100000:1.0:1766099648.966259:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007de97d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432940992:0@lo:400:kworker/u8:2.0
00010000:00080000:1.0:1766099648.966275:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001b04885a x1851885432941248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099655 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099648.966284:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b04885a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432941248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099648.966310:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941184
00000100:00100000:1.0:1766099648.966332:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2569
00000100:00100000:1.0:1766099648.966336:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090ca65cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432941184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099648.966381:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090ca65cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432941184:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (558us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099648.966388:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2569
00000100:00100000:1.0:1766099648.966413:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432941184:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099648.966820:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004bd75cc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432941056:12345-0@lo:400:kworker/u8:2.0 Request processed in 2891us (7518us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099648.966829:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2568
00000100:00100000:3.0:1766099648.967672:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e67a7300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432941056:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099649.472692:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099650.499431:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099651.517717:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099652.541697:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099653.405034:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099653.405095:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099653.405131:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226688
00000100:00100000:1.0:1766099653.405151:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099653.405178:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226624
00000100:00100000:3.0:1766099653.405206:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1595
00000100:00100000:3.0:1766099653.405216:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417226688:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099653.405313:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099653.405393:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226560
00000100:00100000:3.0:1766099653.405407:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417226688:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 190us (383us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099653.405417:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1595
00000100:00100000:2.0:1766099653.405450:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1742
00000100:00100000:2.0:1766099653.405461:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226560:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099653.405569:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226560:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 110us (420us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099653.405578:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1742
00000100:00100000:0.0:1766099653.405743:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2570
00000100:00100000:0.0:1766099653.405751:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417226624:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099653.405808:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b04885a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417226624:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 58us (715us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099653.405815:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2570
00010000:00010000:1.0:1766099653.565713:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099654.080098:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099654.080108:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099654.080113:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099654.080124:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4157
00000100:00080000:2.0:1766099654.080129:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099654.080138:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099654.080161:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432941376:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099654.080184:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003837ea46 x1851885432941440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099654.080265:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5132164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432941504:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099654.080277:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099654.080358:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432941568:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099654.080937:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099654.081060:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432941440:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099654.081060:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941376
00000100:00100000:0.0:1766099654.081108:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099654.081113:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941504
00000100:00100000:2.0:1766099654.081144:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2571
00000100:00100000:2.0:1766099654.081153:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bfca3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432941376:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099654.081161:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941440
00000100:00100000:0.0:1766099654.081197:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1596
00000100:00100000:0.0:1766099654.081208:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed6942f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432941440:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099654.081229:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bfca3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432941376:12345-0@lo:400:kworker/u8:2.0 Request processed in 75us (965us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099654.081240:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2571
00000100:00100000:3.0:1766099654.081268:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a81cfa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432941376:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099654.081269:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2572
00000100:00100000:2.0:1766099654.081278:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432941504:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099654.081344:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432941312:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099654.081345:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388d2359 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432941504:12345-0@lo:400:kworker/u8:2.0 Request processed in 68us (407us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099654.081361:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2572
00000100:00100000:2.0:1766099654.081409:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5132164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432941504:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099654.081414:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099654.081474:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941312
00000100:00100000:2.0:1766099654.081542:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1743
00000100:00100000:2.0:1766099654.081547:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432941312:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099654.081598:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432941312:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (190us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099654.081605:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1743
00000100:00100000:3.0:1766099654.081634:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000612198b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432941312:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099654.081702:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099654.082909:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941568
00000100:00100000:3.0:1766099654.083104:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1135
00000100:00100000:3.0:1766099654.083110:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432941568:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099654.083183:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432941568:12345-0@lo:400:kworker/u8:2.0 Request processed in 73us (1484us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099654.083191:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1135
00010000:00080000:3.0:1766099654.083233:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005dcf37af x1851885432941568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099661 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099654.083245:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432941568:0@lo:400:kworker/u8:2.0
00010000:02020000:0.0:1766099654.083456:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099654.083545:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed6942f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432941440:12345-0@lo:8:kworker/u8:2.0 Request processed in 2339us (2437us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099654.083557:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1596
00000100:00080000:0.0:1766099654.083614:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099654.083618:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099654.083623:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432941440:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099654.590803:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099655.613769:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099656.637896:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099657.661942:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099658.526138:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099658.526187:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099658.526281:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099658.526306:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226816
00000100:00100000:2.0:1766099658.526319:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226752
00000100:00100000:0.0:1766099658.526382:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226880
00000100:00100000:2.0:1766099658.526393:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1744
00000100:00100000:3.0:1766099658.526393:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2573
00000100:00100000:2.0:1766099658.526402:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226752:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099658.526402:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417226816:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099658.526444:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1597
00000100:00100000:0.0:1766099658.526453:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417226880:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766099658.526538:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417226752:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 135us (409us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099658.526539:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082d8c1f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417226816:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 139us (354us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099658.526548:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1744
00000100:00100000:3.0:1766099658.526548:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2573
00010000:02020000:0.0:1766099658.526628:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099658.526688:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417226880:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 234us (412us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099658.526696:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1597
00010000:00010000:1.0:1766099658.686213:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099659.198637:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099659.198646:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099659.198650:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099659.198660:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4162
00000100:00080000:1.0:1766099659.198665:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099659.198671:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099659.198701:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e67a7300 x1851885432941760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099659.199088:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c45bca0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432941824:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.199165:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099659.199200:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099659.199205:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000016c31705 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432941696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.199224:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099659.199256:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941824
00000100:00100000:1.0:1766099659.199307:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941696
00000100:00100000:1.0:1766099659.199327:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2574
00000100:00100000:1.0:1766099659.199333:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abcb39cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432941824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.199382:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abcb39cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432941824:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (218us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099659.199389:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2574
00000100:00100000:1.0:1766099659.199404:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2575
00000100:00100000:1.0:1766099659.199407:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001145b491 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432941696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.199567:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001145b491 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432941696:12345-0@lo:400:kworker/u8:3.0 Request processed in 158us (342us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099659.199573:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2575
00000100:00100000:1.0:1766099659.199610:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000016c31705 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432941696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.199619:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c45bca0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432941824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099659.200032:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432941888:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099659.200070:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099659.200094:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099659.200099:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432941632:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099659.200124:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099659.204295:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e67a7300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432941760:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099659.204336:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941888
00000100:00100000:1.0:1766099659.204393:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099659.204400:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1136
00000100:00100000:0.0:1766099659.204407:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432941888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.204467:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941760
00000100:00100000:0.0:1766099659.204492:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432941888:12345-0@lo:400:kworker/u8:3.0 Request processed in 69us (4406us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099659.204501:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1136
00000100:00100000:1.0:1766099659.204517:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1598
00000100:00100000:1.0:1766099659.204524:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432941760:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099659.204599:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941632
00010000:02020000:1.0:1766099659.204630:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099659.204640:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1745
00000100:00100000:0.0:1766099659.204645:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432941632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099659.204677:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432941760:12345-0@lo:8:kworker/u8:3.0 Request processed in 152us (286us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099659.204688:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1598
00000100:00100000:0.0:1766099659.204689:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432941632:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (4561us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099659.204698:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1745
00000100:00080000:1.0:1766099659.204731:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099659.204736:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099659.204742:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e67a7300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432941760:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099659.205263:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432941632:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099659.205294:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f1197d22 x1851885432941888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099666 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099659.205308:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432941888:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099659.709704:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099660.734119:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099661.758473:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099662.785641:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099663.647549:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099663.647599:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099663.647636:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099663.647716:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417226944
00000100:00100000:0.0:1766099663.647733:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227072
00000100:00100000:3.0:1766099663.647759:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1746
00000100:00100000:0.0:1766099663.647766:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1599
00000100:00100000:3.0:1766099663.647766:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417226944:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099663.647769:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417227072:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:3.0:1766099663.647828:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417226944:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 62us (192us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099663.647836:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1746
00010000:02020000:0.0:1766099663.647859:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099663.647886:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417227072:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 116us (344us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099663.647893:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1599
00000100:00100000:2.0:1766099663.647972:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227008
00000100:00100000:2.0:1766099663.648010:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2576
00000100:00100000:2.0:1766099663.648015:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069dabd45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417227008:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099663.648060:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069dabd45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417227008:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 44us (461us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099663.648067:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2576
00010000:00010000:2.0:1766099663.805719:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099664.317987:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099664.317994:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099664.317997:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099664.318005:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4167
00000100:00080000:3.0:1766099664.318009:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099664.318015:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099664.318035:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bbeb8a65 x1851885432942080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099664.318080:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbeb8a65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432942080:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099664.318142:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099664.318203:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942080
00000100:00100000:3.0:1766099664.318237:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1600
00000100:00100000:3.0:1766099664.318242:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432942080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099664.318361:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099664.318399:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432942080:12345-0@lo:8:kworker/u8:2.0 Request processed in 156us (258us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099664.318406:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1600
00000100:00080000:3.0:1766099664.318438:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099664.318441:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099664.318444:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbeb8a65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432942080:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318528:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432942208:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318559:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099664.318587:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099664.318592:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432941952:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318619:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099664.318643:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432942016:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318667:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099664.318683:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099664.318688:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432942144:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318708:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099664.318734:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942208
00000100:00100000:0.0:1766099664.318763:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1137
00000100:00100000:0.0:1766099664.318767:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000237440eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432942208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318805:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000237440eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432942208:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (246us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099664.318812:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1137
00000100:00100000:0.0:1766099664.318833:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942016
00000100:00100000:0.0:1766099664.318853:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942144
00000100:00100000:0.0:1766099664.318869:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2577
00000100:00100000:0.0:1766099664.318873:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ae9ddfd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432942016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318902:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ae9ddfd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432942016:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (236us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099664.318908:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2577
00000100:00100000:0.0:1766099664.318921:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2578
00000100:00100000:0.0:1766099664.318924:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c81aaacb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432942144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.318947:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c81aaacb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432942144:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (239us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099664.318952:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2578
00000100:00100000:0.0:1766099664.318974:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432941952
00000100:00100000:0.0:1766099664.318994:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1747
00000100:00100000:0.0:1766099664.318997:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432941952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.319020:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432941952:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (402us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099664.319025:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1747
00000100:00100000:0.0:1766099664.319059:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432942144:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.319068:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432942016:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099664.319090:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432941952:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766099664.319098:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000094636470 x1851885432942208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099671 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099664.319107:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432942208:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099664.830299:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099665.853682:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099666.880756:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099667.901740:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099668.764565:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099668.764673:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227264
00000100:00100000:0.0:1766099668.764744:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1601
00000100:00100000:0.0:1766099668.764753:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ef41ef9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417227264:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099668.764869:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099668.764966:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ef41ef9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417227264:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 212us (412us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099668.764974:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1601
00000100:00100000:3.0:1766099668.768730:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099668.768825:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227136
00000100:00100000:0.0:1766099668.768901:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1748
00000100:00100000:0.0:1766099668.768910:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417227136:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099668.769017:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417227136:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 107us (300us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099668.769026:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1748
00000100:00100000:3.0:1766099668.770728:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099668.770799:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227200
00000100:00100000:1.0:1766099668.770863:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2579
00000100:00100000:1.0:1766099668.770870:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbeb8a65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417227200:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099668.770964:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbeb8a65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417227200:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 94us (248us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099668.770973:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2579
00010000:00010000:1.0:1766099668.925721:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099669.439863:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1766099669.439874:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009453e80d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432942272:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099669.439878:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099669.439884:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099669.439895:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4172
00000100:00080000:0.0:1766099669.439898:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099669.439905:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099669.439934:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a71a08db x1851885432942400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099669.439983:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099669.440002:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432942528:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099669.440047:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099669.440057:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942272
00000100:00100000:0.0:1766099669.440087:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b36cfcdc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432942336:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099669.440114:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1749
00000100:00100000:3.0:1766099669.440132:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432942272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099669.440133:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099669.440157:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099669.440172:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8fb2439 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432942464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099669.440197:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432942272:12345-0@lo:400:kworker/u8:3.0 Request processed in 68us (215us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099669.440223:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1749
00000100:00100000:0.0:1766099669.440224:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099669.440264:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a71a08db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432942400:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099669.440289:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009453e80d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432942272:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099669.440296:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099669.440337:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942336
00000100:00100000:3.0:1766099669.440357:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942528
00000100:00100000:0.0:1766099669.440373:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942464
00000100:00100000:3.0:1766099669.440392:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942400
00000100:00100000:0.0:1766099669.440404:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2580
00000100:00100000:0.0:1766099669.440411:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f373762c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432942336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099669.440418:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1138
00000100:00100000:3.0:1766099669.440424:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432942528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099669.440461:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f373762c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432942336:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (333us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099669.440469:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2580
00000100:00100000:3.0:1766099669.440475:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432942528:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (428us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099669.440512:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1138
00000100:00100000:0.0:1766099669.440527:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2581
00000100:00100000:3.0:1766099669.440532:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1602
00000100:00100000:0.0:1766099669.440536:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061def75d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432942464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099669.440541:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079b9f738 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432942400:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099669.440572:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061def75d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432942464:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (366us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099669.440578:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2581
00010000:00080000:0.0:1766099669.440624:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003204340c x1851885432942528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099676 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099669.440636:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432942528:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766099669.440649:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099669.440660:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8fb2439 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432942464:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099669.440674:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b36cfcdc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432942336:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099669.440691:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079b9f738 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432942400:12345-0@lo:8:kworker/u8:3.0 Request processed in 151us (397us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099669.440700:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1602
00000100:00080000:0.0:1766099669.440709:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099669.440715:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099669.440720:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a71a08db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432942400:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766099669.949770:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099670.973830:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099671.998684:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099673.021771:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099673.885958:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099673.886430:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099673.887368:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099673.887642:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227328
00000100:00100000:3.0:1766099673.887704:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1750
00000100:00100000:0.0:1766099673.887881:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227456
00000100:00100000:1.0:1766099673.887964:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227392
00000100:00100000:3.0:1766099673.888029:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417227328:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099673.888117:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417227328:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 89us (1689us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099673.888126:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1750
00000100:00100000:2.0:1766099673.888497:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1603
00000100:00100000:2.0:1766099673.888503:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417227456:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099673.888604:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099673.888653:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417227456:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 149us (2706us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099673.888660:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1603
00000100:00100000:0.0:1766099673.888998:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2582
00000100:00100000:0.0:1766099673.889005:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417227392:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099673.889049:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c823c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417227392:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (1685us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099673.889056:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2582
00010000:00010000:1.0:1766099674.045948:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099674.558189:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bc02f61 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432942592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099674.558297:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099674.558660:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942592
00000100:00100000:2.0:1766099674.558725:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1751
00000100:00100000:2.0:1766099674.558732:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432942592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099674.559197:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432942592:12345-0@lo:400:kworker/u8:3.0 Request processed in 462us (899us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099674.559206:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1751
00000100:00100000:3.0:1766099674.559292:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bc02f61 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432942592:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099674.559678:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099674.559685:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099674.559691:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099674.559697:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4177
00000100:00080000:1.0:1766099674.559702:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099674.559707:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099674.559711:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099674.559725:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432942656:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099674.559736:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c2b7fd72 x1851885432942720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099674.559777:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099674.559801:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432942720:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099674.559806:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432942784:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099674.559849:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099674.559856:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099674.559899:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942656
00000100:00100000:2.0:1766099674.559902:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432942848:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099674.559929:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099674.559944:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942784
00000100:00100000:2.0:1766099674.559963:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942720
00000100:00100000:1.0:1766099674.559974:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2583
00000100:00100000:1.0:1766099674.559980:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a7d46da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432942656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099674.559985:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942848
00000100:00100000:2.0:1766099674.560005:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1139
00000100:00100000:2.0:1766099674.560010:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432942848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099674.560032:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a7d46da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432942656:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (255us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099674.560041:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2583
00000100:00100000:2.0:1766099674.560059:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432942848:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (128us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099674.560063:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2584
00000100:00100000:2.0:1766099674.560069:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1139
00000100:00100000:1.0:1766099674.560072:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432942784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099674.560087:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1604
00000100:00100000:2.0:1766099674.560092:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432942720:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099674.560105:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023de6956 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432942784:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (254us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099674.560111:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2584
00000100:00100000:1.0:1766099674.560152:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432942784:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099674.560163:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432942656:0@lo:400:kworker/u8:3.0
00010000:02020000:2.0:1766099674.560197:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099674.560235:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432942720:12345-0@lo:8:kworker/u8:3.0 Request processed in 142us (387us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099674.560241:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1604
00010000:00080000:2.0:1766099674.560268:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000093f0e26e x1851885432942848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099681 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099674.560277:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432942848:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099674.560315:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099674.560321:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099674.560326:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432942720:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099675.077357:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099676.093789:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099677.119680:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099678.142461:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099679.012727:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099679.012784:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099679.012871:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099679.013174:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227520
00000100:00100000:2.0:1766099679.013238:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1752
00000100:00100000:2.0:1766099679.013244:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417227520:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099679.013314:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417227520:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 69us (529us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099679.013321:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1752
00000100:00100000:1.0:1766099679.013467:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227648
00000100:00100000:1.0:1766099679.013580:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1605
00000100:00100000:1.0:1766099679.013587:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000769cb361 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417227648:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099679.013692:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099679.013769:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000769cb361 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417227648:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 181us (897us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099679.013777:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1605
00000100:00100000:0.0:1766099679.015013:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227584
00000100:00100000:0.0:1766099679.015068:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2585
00000100:00100000:0.0:1766099679.015074:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417227584:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099679.015329:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000848a5086 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417227584:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 254us (2609us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099679.015338:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2585
00010000:00010000:2.0:1766099679.165950:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099679.679119:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099679.679127:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099679.679132:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099679.679154:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4183
00000100:00080000:2.0:1766099679.679158:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099679.679164:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099679.679199:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008358d193 x1851885432943040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099679.679256:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432943104:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679338:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099679.679373:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099679.679380:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432942976:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679403:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099679.679429:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008358d193 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432943040:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679459:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099679.679515:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943104
00000100:00100000:2.0:1766099679.679566:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942976
00000100:00100000:2.0:1766099679.679587:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2586
00000100:00100000:2.0:1766099679.679593:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000612198b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432943104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679649:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000612198b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432943104:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (313us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099679.679655:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2586
00000100:00100000:2.0:1766099679.679671:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2587
00000100:00100000:2.0:1766099679.679675:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a81cfa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432942976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679698:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a81cfa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432942976:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (295us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099679.679704:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2587
00000100:00100000:2.0:1766099679.679730:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943040
00000100:00100000:2.0:1766099679.679751:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1606
00000100:00100000:2.0:1766099679.679755:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005834876b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432943040:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099679.679863:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099679.679899:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005834876b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432943040:12345-0@lo:8:kworker/u8:2.0 Request processed in 142us (440us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099679.679906:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1606
00000100:00080000:2.0:1766099679.679944:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099679.679948:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099679.679952:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008358d193 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432943040:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679984:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432942976:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099679.679992:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003837ea46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432943104:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099679.680100:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432943168:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099679.680144:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099679.680173:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099679.680179:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432942912:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099679.680204:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099679.680296:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432942912
00000100:00100000:3.0:1766099679.680331:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1753
00000100:00100000:3.0:1766099679.680335:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432942912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099679.680371:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432942912:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (167us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099679.680377:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1753
00000100:00100000:3.0:1766099679.680405:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432942912:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099679.681922:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943168
00000100:00100000:3.0:1766099679.682713:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1140
00000100:00100000:3.0:1766099679.682721:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a71a08db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432943168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099679.682812:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a71a08db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432943168:12345-0@lo:400:kworker/u8:2.0 Request processed in 92us (2679us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099679.682820:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1140
00010000:00080000:0.0:1766099679.682924:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005dcf37af x1851885432943168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099686 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099679.682938:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432943168:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099680.190459:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099681.219244:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099682.238574:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099683.263005:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099684.124402:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099684.124448:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099684.124563:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227712
00000100:00100000:1.0:1766099684.124618:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1754
00000100:00100000:1.0:1766099684.124623:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417227712:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099684.124683:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417227712:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 60us (236us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099684.124690:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1754
00000100:00100000:0.0:1766099684.124761:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227776
00000100:00100000:0.0:1766099684.124794:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2588
00000100:00100000:0.0:1766099684.124799:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003837ea46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417227776:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099684.124824:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003837ea46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417227776:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 25us (430us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099684.124830:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2588
00000100:00100000:3.0:1766099684.125472:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099684.125584:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227840
00000100:00100000:2.0:1766099684.125617:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1607
00000100:00100000:2.0:1766099684.125622:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417227840:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099684.125710:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099684.125752:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417227840:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 129us (286us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099684.125758:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1607
00010000:00010000:1.0:1766099684.287048:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099684.801578:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099684.801584:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099684.801587:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099684.801595:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4188
00000100:00080000:1.0:1766099684.801598:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099684.801602:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099684.801618:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000092fe0f9a x1851885432943360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099684.801665:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432943296:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099684.801722:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099684.801745:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099684.801749:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b8df53e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432943424:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099684.801764:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099684.801789:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943296
00000100:00100000:1.0:1766099684.801819:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943424
00000100:00100000:1.0:1766099684.801832:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2589
00000100:00100000:1.0:1766099684.801837:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000677523bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432943296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099684.801874:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000677523bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432943296:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (153us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099684.801881:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2589
00000100:00100000:1.0:1766099684.801893:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2590
00000100:00100000:1.0:1766099684.801896:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3fa8d43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432943424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099684.801912:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3fa8d43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432943424:12345-0@lo:400:kworker/u8:3.0 Request processed in 15us (148us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099684.801917:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2590
00000100:00100000:1.0:1766099684.801943:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b8df53e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432943424:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099684.801952:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432943296:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099684.802028:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432943360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099684.802058:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099684.802116:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432943232:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099684.802144:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099684.802183:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099684.802187:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e100b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432943488:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099684.802205:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099684.802224:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943360
00000100:00100000:2.0:1766099684.802243:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943488
00000100:00100000:2.0:1766099684.802263:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1141
00000100:00100000:2.0:1766099684.802268:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432943488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099684.802301:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432943488:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (94us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099684.802308:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1141
00000100:00100000:2.0:1766099684.802324:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1608
00000100:00100000:2.0:1766099684.802327:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce45f83d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432943360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099684.802433:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099684.802472:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce45f83d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432943360:12345-0@lo:8:kworker/u8:3.0 Request processed in 144us (415us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099684.802500:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1608
00000100:00080000:2.0:1766099684.802533:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099684.802537:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099684.802543:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432943360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099684.802572:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943232
00000100:00100000:2.0:1766099684.802604:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1755
00000100:00100000:2.0:1766099684.802608:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432943232:12345-0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099684.802667:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000004e100b7b x1851885432943488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099691 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099684.802686:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e100b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432943488:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099684.803040:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432943232:12345-0@lo:400:kworker/u8:3.0 Request processed in 429us (893us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099684.803048:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1755
00000100:00100000:3.0:1766099684.804556:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432943232:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766099685.310601:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099686.333730:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099687.360915:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099688.382724:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099689.249654:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099689.249708:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099689.250364:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099689.251082:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227904
00000100:00100000:0.0:1766099689.251142:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1756
00000100:00100000:0.0:1766099689.251150:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417227904:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099689.251239:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417227904:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 89us (1531us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099689.251247:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1756
00000100:00100000:1.0:1766099689.251747:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417227968
00000100:00100000:1.0:1766099689.251791:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2591
00000100:00100000:1.0:1766099689.251796:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071acbd6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417227968:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099689.251841:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071acbd6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417227968:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 45us (1478us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099689.251852:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2591
00000100:00100000:2.0:1766099689.252260:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228032
00000100:00100000:2.0:1766099689.252292:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1609
00000100:00100000:2.0:1766099689.252297:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417228032:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099689.252409:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099689.252941:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417228032:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 641us (3295us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099689.252950:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1609
00010000:00010000:1.0:1766099689.406265:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099689.917777:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099689.917788:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099689.917792:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099689.917803:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4193
00000100:00080000:1.0:1766099689.917807:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099689.917814:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099689.917841:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008038ff3a x1851885432943680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099689.917851:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003b8df53e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432943616:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099689.917942:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432943744:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099689.917944:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099689.918106:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000306edddc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432943808:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099689.918164:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099689.918237:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099689.918262:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099689.918266:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432943552:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099689.918267:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943616
00000100:00100000:0.0:1766099689.918297:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099689.918321:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943744
00000100:00100000:3.0:1766099689.918345:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2592
00000100:00100000:0.0:1766099689.918350:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943552
00000100:00100000:3.0:1766099689.918352:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbb82e8f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432943616:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099689.918376:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1757
00000100:00100000:0.0:1766099689.918380:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432943552:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099689.918408:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432943552:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (110us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099689.918408:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbb82e8f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432943616:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (466us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099689.918418:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1757
00000100:00100000:3.0:1766099689.918418:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2592
00000100:00100000:3.0:1766099689.918434:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2593
00000100:00100000:3.0:1766099689.918437:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cce3c11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432943744:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099689.918462:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cce3c11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432943744:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (300us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099689.918468:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2593
00000100:00100000:2.0:1766099689.918561:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003b8df53e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432943616:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099689.918592:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008038ff3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432943680:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099689.918620:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099689.918653:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943808
00000100:00100000:2.0:1766099689.918691:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943680
00000100:00100000:2.0:1766099689.918917:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1142
00000100:00100000:2.0:1766099689.918924:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b36cfcdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432943808:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099689.918960:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b36cfcdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432943808:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (723us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099689.918966:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1142
00000100:00100000:2.0:1766099689.918980:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1610
00000100:00100000:2.0:1766099689.918983:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432943680:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099689.919315:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432943744:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099689.919634:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432943552:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099689.919650:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000306edddc x1851885432943808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099696 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099689.919660:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000306edddc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432943808:0@lo:400:kworker/u8:3.0
00010000:02020000:2.0:1766099689.919995:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099689.920069:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432943680:12345-0@lo:8:kworker/u8:3.0 Request processed in 1075us (1440us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099689.920084:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1610
00000100:00080000:3.0:1766099689.920949:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099689.920957:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099689.920963:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008038ff3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432943680:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099690.432301:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099691.458185:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099692.477732:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099693.503641:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099694.365621:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099694.365696:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099694.367022:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228096
00000100:00100000:0.0:1766099694.367128:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1758
00000100:00100000:0.0:1766099694.367135:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417228096:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099694.367220:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417228096:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 84us (1524us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099694.367227:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1758
00000100:00100000:1.0:1766099694.367594:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228160
00000100:00100000:1.0:1766099694.367635:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2594
00000100:00100000:1.0:1766099694.367639:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417228160:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099694.367775:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6ad3f27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417228160:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 134us (2162us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099694.367783:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2594
00000100:00100000:3.0:1766099694.371714:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099694.373320:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228224
00000100:00100000:2.0:1766099694.373376:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1611
00000100:00100000:2.0:1766099694.373383:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417228224:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099694.373504:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099694.373582:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417228224:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 198us (1879us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099694.373591:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1611
00010000:00010000:2.0:1766099694.526767:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099695.037808:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766099695.037818:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099695.037827:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099695.037831:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1766099695.037840:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099695.037841:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432943872:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766099695.037842:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4198
00000100:00080000:2.0:1766099695.037852:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099695.037861:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099695.037863:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432943936:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099695.037890:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000041477718 x1851885432944000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099695.037964:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099695.037970:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099695.037979:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009fcfa81e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432944128:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099695.037988:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020194846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432944064:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099695.039828:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099695.039907:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943872
00000100:00100000:0.0:1766099695.039968:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1759
00000100:00100000:0.0:1766099695.039975:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432943872:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099695.040042:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432943872:12345-0@lo:400:kworker/u8:2.0 Request processed in 67us (222us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099695.040050:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1759
00000100:00100000:0.0:1766099695.040111:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432943872:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099695.040162:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944000:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099695.040195:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099695.040505:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944000
00000100:00100000:0.0:1766099695.040526:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1612
00000100:00100000:0.0:1766099695.040530:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8fb2439 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432944000:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099695.040625:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099695.040659:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8fb2439 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432944000:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (464us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099695.040665:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1612
00000100:00080000:0.0:1766099695.042608:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099695.042622:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099695.042632:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944000:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099695.043021:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099695.043389:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099695.043429:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944128
00000100:00100000:1.0:1766099695.043489:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1143
00000100:00100000:1.0:1766099695.043495:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432944128:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099695.043544:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432944128:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (156us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099695.043550:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1143
00010000:00080000:1.0:1766099695.043584:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009fcfa81e x1851885432944128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099702 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099695.043596:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009fcfa81e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432944128:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099695.044879:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099695.044945:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432943936
00000100:00100000:1.0:1766099695.045568:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944064
00000100:00100000:2.0:1766099695.045848:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2595
00000100:00100000:2.0:1766099695.045856:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b84bdab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432943936:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099695.045930:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b84bdab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432943936:12345-0@lo:400:kworker/u8:2.0 Request processed in 75us (2911us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099695.045937:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2595
00000100:00100000:2.0:1766099695.045953:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2596
00000100:00100000:2.0:1766099695.045956:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7865906 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432944064:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099695.045979:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7865906 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432944064:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (1107us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099695.045984:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2596
00000100:00100000:2.0:1766099695.046015:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020194846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432944064:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099695.046490:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432943936:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099695.555062:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099696.575050:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099697.598186:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099698.621722:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099699.488260:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099699.488315:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099699.488356:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228352
00000100:00100000:2.0:1766099699.488382:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228288
00000100:00100000:1.0:1766099699.488404:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099699.488420:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1760
00000100:00100000:3.0:1766099699.488422:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2597
00000100:00100000:2.0:1766099699.488428:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417228288:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099699.488428:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097409eff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417228352:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099699.488461:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228416
00000100:00100000:1.0:1766099699.488512:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1613
00000100:00100000:1.0:1766099699.488522:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228416:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:2.0:1766099699.488542:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417228288:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 112us (227us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099699.488542:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097409eff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417228352:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 112us (291us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099699.488551:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1760
00000100:00100000:3.0:1766099699.488551:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2597
00010000:02020000:1.0:1766099699.488628:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099699.488676:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228416:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 154us (274us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099699.488684:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1613
00010000:00010000:1.0:1766099699.645747:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099700.163975:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099700.163993:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099700.163997:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099700.164007:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4203
00000100:00080000:2.0:1766099700.164012:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099700.164020:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099700.164047:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009ad50fe1 x1851885432944320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099700.164107:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088efffa5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432944256:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099700.164194:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099700.164235:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099700.164241:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432944384:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099700.164266:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099700.164297:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ad50fe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944320:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099700.164326:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099700.164368:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944256
00000100:00100000:2.0:1766099700.164417:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944384
00000100:00100000:2.0:1766099700.164439:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2598
00000100:00100000:2.0:1766099700.164445:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432944256:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099700.164534:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abf3cafe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432944256:12345-0@lo:400:kworker/u8:2.0 Request processed in 88us (342us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099700.164546:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2598
00000100:00100000:3.0:1766099700.164702:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000315ba24d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432944448:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.164735:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099700.164774:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099700.164780:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020194846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432944192:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.164810:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099700.164844:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944320
00000100:00100000:3.0:1766099700.164864:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944448
00000100:00100000:3.0:1766099700.164885:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1144
00000100:00100000:3.0:1766099700.164890:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432944448:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.164925:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432944448:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (190us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099700.164932:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1144
00000100:00100000:3.0:1766099700.164946:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1614
00000100:00100000:3.0:1766099700.164950:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432944320:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766099700.165066:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099700.165103:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432944320:12345-0@lo:8:kworker/u8:2.0 Request processed in 152us (776us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099700.165109:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1614
00000100:00100000:3.0:1766099700.165130:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944192
00000100:00100000:3.0:1766099700.165155:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1761
00000100:00100000:3.0:1766099700.165159:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432944192:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.165189:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432944192:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (378us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099700.165195:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1761
00000100:00100000:3.0:1766099700.165249:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020194846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432944192:0@lo:400:kworker/u8:2.0
00010000:00080000:3.0:1766099700.165262:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000315ba24d x1851885432944448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099707 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099700.165273:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000315ba24d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432944448:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.165426:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2599
00000100:00100000:3.0:1766099700.165432:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432944384:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.165466:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e26029d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432944384:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (1200us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099700.165472:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2599
00000100:00080000:3.0:1766099700.165553:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099700.165558:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099700.166448:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432944384:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099700.166467:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088efffa5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432944256:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099700.179078:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ad50fe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944320:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099700.669735:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099701.694923:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099702.717893:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099703.741743:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099704.606716:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099704.606773:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099704.606901:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099704.607000:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228608
00000100:00100000:3.0:1766099704.607072:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1615
00000100:00100000:3.0:1766099704.607082:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cd4ac16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228608:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099704.607197:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099704.607278:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cd4ac16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228608:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 196us (376us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099704.607287:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1615
00000100:00100000:0.0:1766099704.607518:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228480
00000100:00100000:0.0:1766099704.607563:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1762
00000100:00100000:0.0:1766099704.607567:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417228480:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099704.607610:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417228480:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 41us (908us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099704.607616:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1762
00000100:00100000:2.0:1766099704.607722:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228544
00000100:00100000:2.0:1766099704.607769:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2600
00000100:00100000:2.0:1766099704.607774:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000953f8749 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417228544:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099704.607809:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000953f8749 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417228544:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 35us (1037us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099704.607816:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2600
00010000:00010000:1.0:1766099704.767539:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099705.279568:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099705.279577:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099705.279582:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099705.279592:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4208
00000100:00080000:3.0:1766099705.279596:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099705.279604:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099705.279634:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000bbe45651 x1851885432944640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099705.279691:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944640:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099705.279778:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099705.279888:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432944576:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099705.279927:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099705.279971:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099705.279977:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000769cb361 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432944704:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099705.279999:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099705.280040:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944640
00000100:00100000:1.0:1766099705.280078:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1616
00000100:00100000:1.0:1766099705.280086:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432944640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099705.280182:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099705.280223:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432944640:12345-0@lo:8:kworker/u8:3.0 Request processed in 137us (447us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099705.280231:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1616
00000100:00100000:2.0:1766099705.280655:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944576
00000100:00100000:2.0:1766099705.280705:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944704
00000100:00100000:2.0:1766099705.280728:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2601
00000100:00100000:2.0:1766099705.280732:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432944576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.280787:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000153c8b21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432944576:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (860us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099705.280794:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2601
00000100:00100000:2.0:1766099705.280811:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2602
00000100:00100000:2.0:1766099705.280815:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432944704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.280841:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a150dbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432944704:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (841us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099705.280847:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2602
00000100:00100000:2.0:1766099705.280875:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432944512:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.280910:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099705.280935:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099705.280941:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432944768:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.280979:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099705.281008:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944512
00000100:00100000:2.0:1766099705.281036:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1763
00000100:00100000:2.0:1766099705.281040:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432944512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.281071:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432944512:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (162us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099705.281078:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1763
00000100:00100000:2.0:1766099705.281124:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432944512:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.281158:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000769cb361 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432944704:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.281168:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a593488f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432944576:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.281188:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944768
00000100:00100000:2.0:1766099705.281211:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1145
00000100:00100000:2.0:1766099705.281215:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432944768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099705.281245:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432944768:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (266us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099705.281251:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1145
00010000:00080000:2.0:1766099705.281277:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007e2ba551 x1851885432944768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099712 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099705.281287:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432944768:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099705.284801:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099705.284818:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099705.284828:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbe45651 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944640:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099705.789929:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099706.818326:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099707.838314:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099708.862365:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099709.726777:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099709.726865:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228736
00000100:00100000:1.0:1766099709.726952:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2603
00000100:00100000:1.0:1766099709.726960:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417228736:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099709.727055:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039dc5372 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417228736:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 95us (290us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099709.727064:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2603
00000100:00100000:3.0:1766099709.729555:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099709.729605:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099709.729697:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228800
00000100:00100000:0.0:1766099709.729754:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1617
00000100:00100000:0.0:1766099709.729761:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228800:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099709.729864:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099709.730180:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228672
00000100:00100000:2.0:1766099709.730233:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1764
00000100:00100000:2.0:1766099709.730237:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417228672:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099709.730310:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417228672:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 72us (705us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099709.730317:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1764
00000100:00100000:0.0:1766099709.741863:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228800:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 12093us (12310us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099709.741878:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1617
00010000:00010000:2.0:1766099709.886343:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099710.400374:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099710.400383:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099710.400388:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099710.400398:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4213
00000100:00080000:2.0:1766099710.400403:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099710.400409:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099710.400436:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000088efffa5 x1851885432944960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099710.400524:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ad50fe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432944896:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099710.400603:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099710.400638:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099710.400643:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945024:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099710.400664:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099710.400709:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944896
00000100:00100000:2.0:1766099710.400769:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945024
00000100:00100000:2.0:1766099710.400789:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2604
00000100:00100000:2.0:1766099710.400795:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020194846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432944896:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099710.400936:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020194846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432944896:12345-0@lo:400:kworker/u8:3.0 Request processed in 140us (334us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099710.400945:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2604
00000100:00100000:2.0:1766099710.400969:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2605
00000100:00100000:2.0:1766099710.400975:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fcfa81e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099710.401010:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fcfa81e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945024:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (345us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099710.401016:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2605
00000100:00100000:2.0:1766099710.401079:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945024:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099710.401093:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ad50fe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432944896:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099710.401193:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000315ba24d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432945088:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099710.401227:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088efffa5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944960:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099710.401245:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099710.401270:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099710.401277:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099710.401286:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432944832:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099710.401320:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099710.401331:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945088
00000100:00100000:3.0:1766099710.401361:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944832
00000100:00100000:1.0:1766099710.401368:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432944960
00000100:00100000:1.0:1766099710.401393:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1146
00000100:00100000:1.0:1766099710.401399:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432945088:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099710.401404:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1765
00000100:00100000:3.0:1766099710.401410:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432944832:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099710.402013:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1618
00000100:00100000:0.0:1766099710.402021:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432944960:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099710.402131:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099710.402662:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432944832:12345-0@lo:400:kworker/u8:3.0 Request processed in 1250us (1340us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099710.402672:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1765
00000100:00100000:1.0:1766099710.402821:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432945088:12345-0@lo:400:kworker/u8:3.0 Request processed in 1421us (1580us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099710.402830:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1146
00000100:00100000:0.0:1766099710.404127:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432944960:12345-0@lo:8:kworker/u8:3.0 Request processed in 2098us (2852us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099710.404142:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1618
00000100:00080000:1.0:1766099710.404317:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099710.404322:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099710.404332:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088efffa5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432944960:0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766099710.404967:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000315ba24d x1851885432945088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099717 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099710.404988:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000315ba24d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432945088:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099710.405017:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432944832:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099710.910970:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099711.933740:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099712.958815:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099713.981780:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099714.849317:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099714.849373:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099714.849412:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099714.849523:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228928
00000100:00100000:3.0:1766099714.849584:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2606
00000100:00100000:3.0:1766099714.849591:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000315ba24d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417228928:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099714.849661:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000315ba24d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417228928:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 71us (250us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099714.849669:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2606
00000100:00100000:0.0:1766099714.849750:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228992
00000100:00100000:0.0:1766099714.849784:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1619
00000100:00100000:0.0:1766099714.849788:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228992:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099714.849894:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099714.849926:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417228992:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 136us (617us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099714.849933:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1619
00000100:00100000:1.0:1766099714.850001:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417228864
00000100:00100000:1.0:1766099714.850037:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1766
00000100:00100000:1.0:1766099714.850042:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417228864:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099714.850069:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417228864:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 27us (698us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099714.850076:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1766
00010000:00010000:1.0:1766099715.006465:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099715.517729:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099715.517737:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099715.517741:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099715.517750:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4218
00000100:00080000:1.0:1766099715.517754:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099715.517760:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099715.517783:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002ef41ef9 x1851885432945280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099715.517838:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef41ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432945280:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099715.517902:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432945408:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099715.517933:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099715.517990:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099715.518008:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432945216:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099715.518038:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099715.518052:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014b7c3cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432945152:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099715.518054:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099715.518091:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099715.518100:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945344:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099715.518111:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099715.518133:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099715.518174:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945152
00000100:00100000:1.0:1766099715.518179:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945216
00000100:00100000:1.0:1766099715.518247:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945344
00000100:00100000:2.0:1766099715.518249:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1767
00000100:00100000:2.0:1766099715.518261:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432945152:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099715.518275:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2607
00000100:00100000:1.0:1766099715.518282:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008038ff3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432945216:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099715.518333:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432945152:12345-0@lo:400:kworker/u8:0.0 Request processed in 71us (219us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099715.518346:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008038ff3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432945216:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (295us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099715.518356:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1767
00000100:00100000:1.0:1766099715.518363:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2607
00000100:00100000:1.0:1766099715.518390:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2608
00000100:00100000:1.0:1766099715.518396:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000306edddc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945344:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099715.518411:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945280
00000100:00100000:2.0:1766099715.518445:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014b7c3cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432945152:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099715.518446:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945408
00000100:00100000:2.0:1766099715.518526:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1147
00000100:00100000:2.0:1766099715.518532:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432945408:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099715.519062:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1620
00000100:00100000:3.0:1766099715.519069:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432945280:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099715.519186:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099715.519268:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000306edddc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945344:12345-0@lo:400:kworker/u8:0.0 Request processed in 872us (1137us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099715.519281:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2608
00000100:00100000:0.0:1766099715.520263:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945344:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099715.520293:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432945216:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099715.520382:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432945408:12345-0@lo:400:kworker/u8:0.0 Request processed in 1843us (2390us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099715.520395:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1147
00000100:00100000:3.0:1766099715.520460:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432945280:12345-0@lo:8:kworker/u8:0.0 Request processed in 1387us (2527us total) trans 0 rc -19/-19
00010000:00080000:0.0:1766099715.520473:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000006baa8a6 x1851885432945408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099722 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099715.520474:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1620
00000100:00100000:0.0:1766099715.520523:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432945408:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766099715.520539:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099715.520546:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099715.520553:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef41ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432945280:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766099716.030768:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099717.053670:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099718.078367:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099719.101755:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099719.967516:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099719.967569:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099719.968150:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099719.968519:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229056
00000100:00100000:0.0:1766099719.968569:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1768
00000100:00100000:0.0:1766099719.968575:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417229056:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099719.968644:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417229056:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (1075us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099719.968652:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1768
00000100:00100000:3.0:1766099719.969237:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229184
00000100:00100000:3.0:1766099719.969275:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1621
00000100:00100000:3.0:1766099719.969280:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417229184:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099719.969372:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099719.969861:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229120
00000100:00100000:1.0:1766099719.969897:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2609
00000100:00100000:1.0:1766099719.969902:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088efffa5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417229120:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099719.970167:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417229184:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 886us (2659us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099719.970176:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1621
00000100:00100000:1.0:1766099719.970216:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088efffa5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417229120:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 312us (2066us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099719.970222:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2609
00010000:00010000:1.0:1766099720.125822:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099720.637797:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099720.637804:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099720.637808:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099720.637815:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4223
00000100:00080000:0.0:1766099720.637818:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099720.637824:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099720.637857:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f78b201d x1851885432945600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099720.637913:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432945536:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099720.637972:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099720.638002:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099720.638007:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945664:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099720.638027:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099720.638056:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945536
00000100:00100000:0.0:1766099720.638096:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945664
00000100:00100000:0.0:1766099720.638114:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2610
00000100:00100000:0.0:1766099720.638119:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed6942f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432945536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099720.638161:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed6942f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432945536:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (190us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099720.638168:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2610
00000100:00100000:0.0:1766099720.638187:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2611
00000100:00100000:0.0:1766099720.638190:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000237440eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099720.638212:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000237440eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945664:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (185us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099720.638217:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2611
00000100:00100000:0.0:1766099720.638252:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945664:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099720.638262:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432945536:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099720.638375:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432945472:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099720.638409:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099720.638434:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099720.638438:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009fb1deee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432945728:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099720.638460:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099720.638552:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945728
00000100:00100000:0.0:1766099720.638590:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1148
00000100:00100000:0.0:1766099720.638594:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432945728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099720.638634:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432945728:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (174us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099720.638640:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1148
00010000:00080000:0.0:1766099720.638664:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009fb1deee x1851885432945728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099727 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099720.638672:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009fb1deee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432945728:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099720.638736:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945472
00000100:00100000:3.0:1766099720.638769:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1769
00000100:00100000:3.0:1766099720.638773:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432945472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099720.638811:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432945472:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (402us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099720.638818:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1769
00000100:00100000:3.0:1766099720.638859:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432945472:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099720.639073:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f78b201d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432945600:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099720.639104:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099720.639235:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945600
00000100:00100000:1.0:1766099720.639261:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1622
00000100:00100000:1.0:1766099720.639264:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000394ecbcb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432945600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099720.639356:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099720.639398:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000394ecbcb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432945600:12345-0@lo:8:kworker/u8:0.0 Request processed in 132us (293us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099720.639404:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1622
00000100:00080000:3.0:1766099720.639507:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099720.639512:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099720.639515:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f78b201d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432945600:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766099721.149714:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099722.173716:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099723.198752:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099724.221776:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099725.085375:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099725.085519:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229312
00000100:00100000:2.0:1766099725.085724:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2612
00000100:00100000:2.0:1766099725.085732:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fb1deee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417229312:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099725.087368:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fb1deee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417229312:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1629us (1995us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099725.087382:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2612
00000100:00100000:1.0:1766099725.087788:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099725.087947:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099725.087979:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229248
00000100:00100000:1.0:1766099725.088014:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1770
00000100:00100000:1.0:1766099725.088019:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417229248:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099725.088083:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417229248:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 63us (137us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099725.088095:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1770
00000100:00100000:2.0:1766099725.091213:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229376
00000100:00100000:2.0:1766099725.091279:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1623
00000100:00100000:2.0:1766099725.091286:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417229376:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766099725.091384:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099725.091474:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417229376:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 187us (3686us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099725.091507:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1623
00010000:00010000:2.0:1766099725.245696:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099725.757730:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766099725.757737:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099725.757742:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:2.0:1766099725.757745:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432945792:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766099725.757745:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099725.757750:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4229
00000100:00080000:3.0:1766099725.757753:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099725.757757:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099725.757774:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000413b47c6 x1851885432945920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099725.757821:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432945856:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099725.757841:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099725.757850:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099725.757866:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099725.757867:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432946048:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099725.757870:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945984:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099725.757889:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099725.757890:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099725.757910:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432945920:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099725.757920:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945792
00000100:00100000:3.0:1766099725.757929:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099725.757952:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1771
00000100:00100000:3.0:1766099725.757954:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945856
00000100:00100000:2.0:1766099725.757958:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432945792:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099725.757977:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945984
00000100:00100000:3.0:1766099725.757993:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2613
00000100:00100000:3.0:1766099725.757997:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432945856:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099725.757998:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432945792:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (158us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099725.758004:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1771
00000100:00100000:3.0:1766099725.758024:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a593488f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432945856:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (173us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099725.758030:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2613
00000100:00100000:2.0:1766099725.758033:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2614
00000100:00100000:2.0:1766099725.758036:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000769cb361 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945984:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099725.758059:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000769cb361 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432945984:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (170us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099725.758065:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2614
00000100:00100000:2.0:1766099725.758082:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946048
00000100:00100000:2.0:1766099725.758100:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432945920
00000100:00100000:2.0:1766099725.758112:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1149
00000100:00100000:2.0:1766099725.758115:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432946048:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099725.758139:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432946048:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (248us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099725.758144:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1149
00000100:00100000:2.0:1766099725.758154:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1624
00000100:00100000:2.0:1766099725.758157:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432945920:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099725.758159:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007382a6ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432945984:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099725.758167:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de913b58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432945856:0@lo:400:kworker/u8:2.0
00010000:02020000:2.0:1766099725.758242:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099725.758269:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432945920:12345-0@lo:8:kworker/u8:2.0 Request processed in 111us (339us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099725.758274:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1624
00000100:00080000:2.0:1766099725.758296:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099725.758299:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099725.758302:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432945920:0@lo:8:kworker/u8:2.0
00010000:00080000:2.0:1766099725.758321:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007e2ba551 x1851885432946048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099732 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099725.758329:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432946048:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099725.758335:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432945792:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099726.269721:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099727.295502:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099728.317719:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099729.342116:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099730.207022:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099730.207077:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099730.207137:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229440
00000100:00100000:1.0:1766099730.207145:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099730.207213:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1772
00000100:00100000:2.0:1766099730.207223:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417229440:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099730.207304:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417229440:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 83us (295us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099730.207312:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1772
00000100:00100000:0.0:1766099730.207410:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229568
00000100:00100000:0.0:1766099730.207442:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1625
00000100:00100000:0.0:1766099730.207447:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417229568:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099730.207578:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099730.207611:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417229568:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 163us (470us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099730.207619:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1625
00000100:00100000:3.0:1766099730.208040:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229504
00000100:00100000:3.0:1766099730.208108:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2615
00000100:00100000:3.0:1766099730.208115:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b8df53e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417229504:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099730.208195:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b8df53e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417229504:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 79us (1119us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099730.208203:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2615
00010000:00010000:1.0:1766099730.367151:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099730.877923:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099730.877932:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099730.877936:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099730.877946:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4234
00000100:00080000:0.0:1766099730.877950:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099730.877956:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099730.877979:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000afd0d51e x1851885432946240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099730.878043:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000079b9f738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432946304:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099730.878085:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afd0d51e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432946240:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099730.878337:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432946112:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099730.886127:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099730.886204:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099730.886220:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432946176:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099730.886295:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099730.886339:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a71a08db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432946368:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099730.886370:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099730.886418:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946304
00000100:00100000:0.0:1766099730.886465:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946176
00000100:00100000:0.0:1766099730.886519:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2616
00000100:00100000:0.0:1766099730.886525:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b36cfcdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432946304:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099730.886599:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b36cfcdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432946304:12345-0@lo:400:kworker/u8:2.0 Request processed in 73us (483us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099730.886607:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2616
00000100:00100000:0.0:1766099730.886627:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2617
00000100:00100000:0.0:1766099730.886630:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8fb2439 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432946176:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099730.886656:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8fb2439 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432946176:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (362us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099730.886662:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2617
00000100:00100000:0.0:1766099730.886704:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432946176:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099730.886718:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000079b9f738 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432946304:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099730.892265:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946368
00000100:00100000:3.0:1766099730.892364:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1150
00000100:00100000:3.0:1766099730.892372:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009453e80d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432946368:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099730.892553:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009453e80d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432946368:12345-0@lo:400:kworker/u8:2.0 Request processed in 180us (6182us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099730.892561:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1150
00000100:00100000:1.0:1766099730.893954:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:0.0:1766099730.893955:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a71a08db x1851885432946368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099737 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099730.893976:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a71a08db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432946368:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099730.894173:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946112
00000100:00100000:3.0:1766099730.894213:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1773
00000100:00100000:3.0:1766099730.894218:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432946112:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099730.894276:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432946112:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (328us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099730.894283:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1773
00000100:00100000:0.0:1766099730.894325:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432946112:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099730.894387:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099730.894795:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946240
00000100:00100000:0.0:1766099730.894821:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1626
00000100:00100000:0.0:1766099730.894825:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d271259f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432946240:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099730.894924:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099730.894963:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d271259f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432946240:12345-0@lo:8:kworker/u8:2.0 Request processed in 138us (577us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099730.894969:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1626
00000100:00080000:3.0:1766099730.895805:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099730.895814:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099730.895819:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afd0d51e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432946240:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099731.390128:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099732.415011:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099733.442334:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099734.462967:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099735.325006:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099735.325059:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099735.325100:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099735.325693:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229696
00000100:00100000:3.0:1766099735.325783:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2618
00000100:00100000:3.0:1766099735.325793:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e100b7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417229696:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099735.325910:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e100b7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417229696:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 118us (914us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099735.325917:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2618
00000100:00100000:0.0:1766099735.326535:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229632
00000100:00100000:2.0:1766099735.326570:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229760
00000100:00100000:0.0:1766099735.326589:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1774
00000100:00100000:0.0:1766099735.326595:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417229632:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099735.326606:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1627
00000100:00100000:2.0:1766099735.326612:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092fe0f9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417229760:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766099735.326641:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417229632:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (1582us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099735.326649:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1774
00010000:02020000:2.0:1766099735.326707:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099735.326772:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092fe0f9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417229760:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 159us (1671us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099735.326780:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1627
00010000:00010000:1.0:1766099735.487092:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099736.001285:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099736.001295:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099736.001300:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099736.001310:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4239
00000100:00080000:1.0:1766099736.001314:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099736.001320:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099736.001350:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002ef41ef9 x1851885432946560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099736.001446:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bc02f61 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432946496:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.001567:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099736.001601:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099736.001607:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432946624:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.001630:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099736.001657:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef41ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432946560:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099736.001684:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099736.001734:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946496
00000100:00100000:3.0:1766099736.001789:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946624
00000100:00100000:3.0:1766099736.001810:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2619
00000100:00100000:3.0:1766099736.001818:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de913b58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432946496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.001877:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de913b58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432946496:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (311us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099736.001885:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2619
00000100:00100000:3.0:1766099736.001903:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2620
00000100:00100000:3.0:1766099736.001907:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432946624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.001929:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007382a6ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432946624:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (299us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099736.001934:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2620
00000100:00100000:3.0:1766099736.001954:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946560
00000100:00100000:3.0:1766099736.001971:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1628
00000100:00100000:3.0:1766099736.001975:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432946560:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099736.002082:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099736.002117:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432946560:12345-0@lo:8:kworker/u8:0.0 Request processed in 141us (432us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099736.002123:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1628
00000100:00080000:3.0:1766099736.002161:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099736.002165:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099736.002168:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef41ef9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432946560:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099736.002202:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432946624:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.002214:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bc02f61 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432946496:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099736.002509:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432946688:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099736.002549:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099736.002577:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099736.002583:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432946432:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099736.002616:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099736.002712:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946688
00000100:00100000:0.0:1766099736.002765:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1151
00000100:00100000:0.0:1766099736.002770:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432946688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099736.002816:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432946688:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (267us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099736.002822:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1151
00010000:00080000:0.0:1766099736.002859:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000069d150bd x1851885432946688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099743 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099736.002872:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432946688:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.003007:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946432
00000100:00100000:3.0:1766099736.003037:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1775
00000100:00100000:3.0:1766099736.003041:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432946432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099736.003325:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432946432:12345-0@lo:400:kworker/u8:0.0 Request processed in 282us (709us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099736.003332:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1775
00000100:00100000:2.0:1766099736.003385:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432946432:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099736.510955:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099737.534337:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099738.557735:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099739.583349:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099740.446831:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099740.446884:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099740.446924:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099740.446961:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229888
00000100:00100000:0.0:1766099740.447009:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229952
00000100:00100000:0.0:1766099740.447046:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1629
00000100:00100000:1.0:1766099740.447050:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2621
00000100:00100000:0.0:1766099740.447055:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417229952:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:1.0:1766099740.447059:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041ad3224 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417229888:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099740.447161:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041ad3224 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417229888:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 103us (277us total) trans 0 rc 0/0
00010000:02020000:0.0:1766099740.447172:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099740.447173:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2621
00000100:00100000:0.0:1766099740.447208:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417229952:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 153us (284us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099740.447215:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1629
00000100:00100000:3.0:1766099740.447337:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417229824
00000100:00100000:3.0:1766099740.447376:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1776
00000100:00100000:3.0:1766099740.447381:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417229824:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099740.447431:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417229824:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 49us (614us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099740.447438:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1776
00010000:00010000:2.0:1766099740.605807:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099741.118242:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099741.118251:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099741.118256:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099741.118267:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4244
00000100:00080000:3.0:1766099741.118271:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099741.118284:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099741.118312:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000096d55fda x1851885432946880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099741.118358:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432946816:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.118438:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099741.118468:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099741.118472:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000772d840f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432946944:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.118524:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099741.118547:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432946880:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099741.118571:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099741.118606:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946816
00000100:00100000:3.0:1766099741.118649:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946944
00000100:00100000:3.0:1766099741.118675:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2622
00000100:00100000:3.0:1766099741.118681:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e44afcc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432946816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.118738:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e44afcc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432946816:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (302us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099741.118745:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2622
00000100:00100000:3.0:1766099741.118767:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2623
00000100:00100000:3.0:1766099741.118770:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de0c7c85 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432946944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.118790:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de0c7c85 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432946944:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (266us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099741.118795:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2623
00000100:00100000:3.0:1766099741.118841:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000772d840f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432946944:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.118852:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432946816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099741.118929:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432946752:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099741.118967:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099741.118998:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099741.119003:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432947008:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099741.119024:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099741.119054:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946880
00000100:00100000:0.0:1766099741.119075:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947008
00000100:00100000:0.0:1766099741.119097:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1152
00000100:00100000:0.0:1766099741.119100:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000665bb208 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432947008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099741.119133:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000665bb208 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432947008:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (108us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099741.119138:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1152
00000100:00100000:0.0:1766099741.119154:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1630
00000100:00100000:0.0:1766099741.119157:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432946880:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099741.119260:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099741.119304:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432946880:12345-0@lo:8:kworker/u8:2.0 Request processed in 145us (732us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099741.119316:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1630
00000100:00080000:0.0:1766099741.119352:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099741.119356:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099741.119360:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432946880:0@lo:8:kworker/u8:2.0
00010000:00080000:0.0:1766099741.119384:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006de96a83 x1851885432947008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099748 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099741.119398:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432947008:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.119472:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432946752
00000100:00100000:3.0:1766099741.119530:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1777
00000100:00100000:3.0:1766099741.119534:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432946752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099741.119568:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432946752:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (602us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099741.119575:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1777
00000100:00100000:0.0:1766099741.119653:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432946752:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099741.639091:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099742.656394:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099743.677828:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099744.704792:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099745.564835:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099745.564917:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099745.565039:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230016
00000100:00100000:3.0:1766099745.565068:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230080
00000100:00100000:0.0:1766099745.565116:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1778
00000100:00100000:3.0:1766099745.565123:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2624
00000100:00100000:0.0:1766099745.565128:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417230016:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099745.565132:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417230080:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099745.565220:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe333148 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417230080:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (303us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099745.565233:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2624
00000100:00100000:0.0:1766099745.565262:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417230016:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 135us (438us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099745.565271:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1778
00000100:00100000:1.0:1766099745.567668:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099745.567984:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230144
00000100:00100000:3.0:1766099745.568045:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1631
00000100:00100000:3.0:1766099745.568054:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417230144:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099745.568165:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099745.568251:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417230144:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 197us (605us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099745.568260:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1631
00010000:00010000:1.0:1766099745.725742:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099746.240023:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099746.240033:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099746.240037:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099746.240049:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4249
00000100:00100000:1.0:1766099746.240051:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766099746.240054:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099746.240062:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099746.240074:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000772d840f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432947072:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099746.240089:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000084b59448 x1851885432947200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099746.240146:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c1aab37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432947136:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099746.240164:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099746.240193:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099746.240198:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de379e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432947328:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099746.240226:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099746.240227:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099746.240235:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000399d91fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432947264:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099746.240266:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947072
00000100:00100000:3.0:1766099746.240267:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099746.240307:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084b59448 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432947200:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099746.240317:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1779
00000100:00100000:1.0:1766099746.240325:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432947072:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099746.240335:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099746.240376:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947136
00000100:00100000:1.0:1766099746.240383:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432947072:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (221us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099746.240390:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947264
00000100:00100000:1.0:1766099746.240392:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1779
00000100:00100000:3.0:1766099746.240415:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2625
00000100:00100000:3.0:1766099746.240422:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eaec260 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432947136:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099746.240425:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2626
00000100:00100000:0.0:1766099746.240432:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1968617 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432947264:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099746.240445:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000772d840f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432947072:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099746.240499:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eaec260 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432947136:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (283us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099746.240507:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2625
00000100:00100000:1.0:1766099746.240522:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947328
00000100:00100000:1.0:1766099746.240553:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947200
00000100:00100000:1.0:1766099746.240572:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1153
00000100:00100000:3.0:1766099746.240574:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c1aab37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432947136:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099746.240579:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432947328:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099746.240617:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632
00000100:00100000:1.0:1766099746.240618:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432947328:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (390us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099746.240628:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1153
00000100:00100000:3.0:1766099746.240628:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432947200:12345-0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766099746.240663:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000de379e43 x1851885432947328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099753 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099746.240673:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de379e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432947328:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099746.240746:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1968617 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432947264:12345-0@lo:400:kworker/u8:2.0 Request processed in 312us (479us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099746.240753:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2626
00000100:00100000:2.0:1766099746.240831:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000399d91fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432947264:0@lo:400:kworker/u8:2.0
00010000:02020000:3.0:1766099746.242177:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099746.242302:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432947200:12345-0@lo:8:kworker/u8:2.0 Request processed in 1674us (1966us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099746.242317:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632
00000100:00080000:0.0:1766099746.242658:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099746.242665:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099746.242672:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084b59448 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432947200:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099746.749801:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099747.774335:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099748.802609:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099749.825324:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099750.688101:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099750.698812:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230336
00000100:00100000:2.0:1766099750.698934:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1633
00000100:00100000:2.0:1766099750.698943:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417230336:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099750.699074:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099750.699193:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417230336:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 249us (11103us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099750.699203:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1633
00000100:00100000:0.0:1766099750.699652:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099750.699722:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099750.699930:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230272
00000100:00100000:1.0:1766099750.700008:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2627
00000100:00100000:1.0:1766099750.700015:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a71a08db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417230272:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099750.700099:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a71a08db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417230272:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 83us (377us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099750.700107:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2627
00000100:00100000:2.0:1766099750.700391:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230208
00000100:00100000:2.0:1766099750.700978:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1780
00000100:00100000:2.0:1766099750.700987:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417230208:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099750.701077:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417230208:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 91us (1427us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099750.701085:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1780
00010000:00010000:1.0:1766099750.845937:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099751.359663:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099751.359670:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099751.359673:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099751.359681:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4254
00000100:00080000:2.0:1766099751.359683:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099751.359689:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099751.359691:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ad805d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432947392:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099751.359707:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000c488766 x1851885432947520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099751.359759:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099751.359766:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005f676fcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432947456:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099751.359791:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099751.359795:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bedb856 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432947648:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099751.359797:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099751.359816:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099751.359818:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099751.359820:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007626dec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432947584:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099751.359850:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947392
00000100:00100000:2.0:1766099751.359850:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099751.359873:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c488766 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432947520:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099751.359890:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1781
00000100:00100000:2.0:1766099751.359894:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099751.359895:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432947392:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099751.359920:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947456
00000100:00100000:1.0:1766099751.359938:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432947392:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (181us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099751.359944:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947584
00000100:00100000:1.0:1766099751.359945:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1781
00000100:00100000:2.0:1766099751.359958:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2628
00000100:00100000:2.0:1766099751.359962:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004ba22e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432947456:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099751.359972:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2629
00000100:00100000:1.0:1766099751.359976:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7b70f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432947584:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099751.359992:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004ba22e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432947456:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (195us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099751.359996:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7b70f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432947584:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (146us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099751.359998:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2628
00000100:00100000:1.0:1766099751.360001:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2629
00000100:00100000:2.0:1766099751.360018:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947648
00000100:00100000:1.0:1766099751.360030:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ad805d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432947392:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099751.360035:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947520
00000100:00100000:2.0:1766099751.360048:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1154
00000100:00100000:1.0:1766099751.360052:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1634
00000100:00100000:2.0:1766099751.360052:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432947648:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099751.360056:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432947520:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099751.360079:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432947648:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (261us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099751.360085:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1154
00000100:00100000:2.0:1766099751.360106:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007626dec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432947584:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099751.360113:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005f676fcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432947456:0@lo:400:kworker/u8:0.0
00010000:02020000:1.0:1766099751.360147:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099751.360176:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432947520:12345-0@lo:8:kworker/u8:0.0 Request processed in 119us (281us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099751.360183:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1634
00000100:00080000:1.0:1766099751.360203:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099751.360206:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099751.360210:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c488766 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432947520:0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766099751.360229:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008bedb856 x1851885432947648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099758 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099751.360236:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bedb856 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432947648:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099751.873272:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099752.898905:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099753.918468:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099754.941771:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099755.807035:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099755.807101:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099755.807159:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099755.807283:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230528
00000100:00100000:2.0:1766099755.807345:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1635
00000100:00100000:2.0:1766099755.807353:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065fe2a9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417230528:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099755.807458:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099755.807571:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065fe2a9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417230528:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 216us (410us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099755.807590:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1635
00000100:00100000:3.0:1766099755.807927:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230464
00000100:00100000:3.0:1766099755.807979:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2630
00000100:00100000:3.0:1766099755.807984:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079b9f738 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417230464:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099755.808038:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079b9f738 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417230464:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 53us (1016us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099755.808045:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2630
00000100:00100000:1.0:1766099755.808286:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230400
00000100:00100000:1.0:1766099755.808351:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1782
00000100:00100000:1.0:1766099755.808356:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417230400:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099755.808395:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417230400:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 39us (1295us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099755.808402:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1782
00010000:00010000:2.0:1766099755.968469:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099756.482566:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099756.482584:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099756.482589:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099756.482600:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4259
00000100:00080000:2.0:1766099756.482603:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099756.482609:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099756.482637:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004cb5390f x1851885432947840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099756.482707:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007626dec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432947776:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099756.482790:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099756.482828:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099756.482834:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432947904:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099756.482856:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099756.482903:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947776
00000100:00100000:2.0:1766099756.482959:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947904
00000100:00100000:2.0:1766099756.482984:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2631
00000100:00100000:2.0:1766099756.482991:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432947776:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099756.483049:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea8a9953 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432947776:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (259us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099756.483057:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2631
00000100:00100000:2.0:1766099756.483077:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2632
00000100:00100000:2.0:1766099756.483080:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432947904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099756.483104:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc68e892 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432947904:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (248us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099756.483109:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2632
00000100:00100000:2.0:1766099756.483157:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004722f0a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432947904:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099756.483167:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007626dec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432947776:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099756.483210:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432947840:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099756.483240:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099756.483849:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f31300c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432947968:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099756.483888:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099756.483922:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099756.483928:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005f676fcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432947712:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099756.483960:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099756.484004:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947840
00000100:00100000:0.0:1766099756.484030:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947968
00000100:00100000:0.0:1766099756.484053:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1155
00000100:00100000:0.0:1766099756.484056:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432947968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099756.484094:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432947968:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (206us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099756.484101:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1155
00000100:00100000:0.0:1766099756.484117:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1636
00000100:00100000:0.0:1766099756.484120:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007626dec1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432947840:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099756.484215:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099756.484247:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007626dec1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432947840:12345-0@lo:8:kworker/u8:2.0 Request processed in 126us (1007us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099756.484254:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1636
00000100:00080000:0.0:1766099756.484293:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099756.484298:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099756.484302:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432947840:0@lo:8:kworker/u8:2.0
00010000:00080000:0.0:1766099756.484329:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003f31300c x1851885432947968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099763 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766099756.484339:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f31300c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432947968:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099756.485544:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432947712
00000100:00100000:1.0:1766099756.485615:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1783
00000100:00100000:1.0:1766099756.485621:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432947712:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099756.485692:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432947712:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (1732us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099756.485699:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1783
00000100:00100000:0.0:1766099756.487758:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005f676fcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432947712:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099756.991076:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099758.013681:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099759.037808:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099760.061777:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099760.925609:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099760.925669:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099760.925753:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230592
00000100:00100000:2.0:1766099760.925788:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099760.925818:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1784
00000100:00100000:0.0:1766099760.925826:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417230592:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099760.925885:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230656
00000100:00100000:3.0:1766099760.925929:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2633
00000100:00100000:3.0:1766099760.925934:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004722f0a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417230656:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099760.926014:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417230592:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 187us (344us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099760.926023:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1784
00000100:00100000:3.0:1766099760.926320:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004722f0a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417230656:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 384us (724us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099760.926332:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2633
00000100:00100000:3.0:1766099760.929653:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230720
00000100:00100000:3.0:1766099760.929728:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1637
00000100:00100000:3.0:1766099760.929736:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417230720:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099760.929845:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099760.929958:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417230720:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 223us (4171us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099760.929966:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1637
00010000:00010000:1.0:1766099761.085763:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099761.600830:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099761.600839:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099761.600844:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099761.600855:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4264
00000100:00080000:0.0:1766099761.600859:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099761.600866:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099761.600895:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b5132164 x1851885432948160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099761.600896:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432948096:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099761.600939:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005834876b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432948288:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099761.601043:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f78b201d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432948032:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099761.601392:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5132164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432948160:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099761.601712:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099761.601752:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099761.601758:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a6cc55b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432948224:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099761.601782:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099761.601786:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099761.601828:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948288
00000100:00100000:2.0:1766099761.601828:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948096
00000100:00100000:2.0:1766099761.601869:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948224
00000100:00100000:0.0:1766099761.601872:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1156
00000100:00100000:0.0:1766099761.601879:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432948288:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099761.601883:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2634
00000100:00100000:2.0:1766099761.601887:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432948096:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099761.601930:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432948288:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (148us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099761.601930:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3227a74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432948096:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (220us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099761.601939:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1156
00000100:00100000:2.0:1766099761.601939:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2634
00000100:00100000:2.0:1766099761.601956:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2635
00000100:00100000:2.0:1766099761.601959:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432948224:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099761.601982:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dd103c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432948224:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (196us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099761.601988:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2635
00010000:00080000:0.0:1766099761.601997:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005834876b x1851885432948288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099768 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099761.602012:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005834876b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432948288:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099761.602021:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a6cc55b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432948224:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099761.602030:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432948096:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099761.602335:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099761.602438:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948032
00000100:00100000:2.0:1766099761.602503:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1785
00000100:00100000:2.0:1766099761.602507:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432948032:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099761.602559:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432948032:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (225us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099761.602565:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1785
00000100:00100000:3.0:1766099761.602651:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f78b201d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432948032:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099761.603145:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099761.603229:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948160
00000100:00100000:2.0:1766099761.603259:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1638
00000100:00100000:2.0:1766099761.603263:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432948160:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766099761.603361:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099761.603408:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432948160:12345-0@lo:8:kworker/u8:0.0 Request processed in 144us (264us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099761.603415:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1638
00000100:00080000:2.0:1766099761.603449:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099761.603454:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099761.603458:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5132164 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432948160:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766099762.109732:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099763.134418:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099764.161032:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099765.184143:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099766.045388:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099766.045447:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099766.045553:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099766.045643:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230848
00000100:00100000:0.0:1766099766.045714:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2636
00000100:00100000:0.0:1766099766.045722:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417230848:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099766.045810:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa4b7e05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417230848:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 88us (434us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099766.045818:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2636
00000100:00100000:1.0:1766099766.047193:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230912
00000100:00100000:1.0:1766099766.047252:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1639
00000100:00100000:1.0:1766099766.047260:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417230912:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:3.0:1766099766.047272:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230784
00000100:00100000:3.0:1766099766.047324:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1786
00000100:00100000:3.0:1766099766.047330:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417230784:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00010000:02020000:1.0:1766099766.047362:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099766.047416:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417230784:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 85us (1968us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099766.047424:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1786
00000100:00100000:1.0:1766099766.052286:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417230912:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 5013us (6720us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099766.052311:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1639
00010000:00010000:1.0:1766099766.206770:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099766.719612:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099766.719623:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099766.719627:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099766.719638:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4270
00000100:00080000:2.0:1766099766.719642:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099766.719650:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099766.719676:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003f31300c x1851885432948480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099766.719727:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005f676fcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432948416:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099766.719811:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099766.719846:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099766.719852:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432948544:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099766.719875:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099766.719906:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f31300c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432948480:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099766.719940:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099766.719981:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948416
00000100:00100000:2.0:1766099766.720028:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948544
00000100:00100000:2.0:1766099766.720050:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2637
00000100:00100000:2.0:1766099766.720056:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c488766 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432948416:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099766.720115:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c488766 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432948416:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (305us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099766.720123:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2637
00000100:00100000:2.0:1766099766.720141:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2638
00000100:00100000:2.0:1766099766.720144:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ad805d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432948544:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099766.720167:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ad805d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432948544:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (292us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099766.720174:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2638
00000100:00100000:2.0:1766099766.720194:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948480
00000100:00100000:2.0:1766099766.720216:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1640
00000100:00100000:2.0:1766099766.720220:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d472cad pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432948480:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766099766.720316:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099766.720350:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d472cad pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432948480:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (410us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099766.720356:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1640
00000100:00080000:2.0:1766099766.720586:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099766.720591:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099766.720595:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f31300c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432948480:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099766.720629:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432948544:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099766.720640:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005f676fcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432948416:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099766.720735:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432948352:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099766.720775:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099766.720810:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099766.720815:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bedb856 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432948608:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099766.720842:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099766.721026:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948352
00000100:00100000:2.0:1766099766.721064:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1787
00000100:00100000:2.0:1766099766.721069:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432948352:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099766.721190:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948608
00000100:00100000:3.0:1766099766.721231:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1157
00000100:00100000:3.0:1766099766.721234:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432948608:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099766.723251:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432948608:12345-0@lo:400:kworker/u8:2.0 Request processed in 2004us (2398us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099766.723267:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1157
00000100:00100000:2.0:1766099766.723363:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432948352:12345-0@lo:400:kworker/u8:2.0 Request processed in 2293us (2587us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099766.723373:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1787
00000100:00100000:1.0:1766099766.723473:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000daafc58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432948352:0@lo:400:kworker/u8:2.0
00010000:00080000:1.0:1766099766.723542:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008bedb856 x1851885432948608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099773 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099766.723555:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bedb856 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432948608:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099767.231873:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099768.253796:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099769.277755:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099770.304385:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099771.164446:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099771.164559:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099771.164686:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417230976
00000100:00100000:3.0:1766099771.164756:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1788
00000100:00100000:3.0:1766099771.164764:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417230976:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099771.164872:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417230976:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 107us (313us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099771.164880:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1788
00000100:00100000:0.0:1766099771.165143:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231040
00000100:00100000:0.0:1766099771.165189:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2639
00000100:00100000:0.0:1766099771.165194:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f676fcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417231040:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099771.165242:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f676fcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417231040:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 47us (808us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099771.165250:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2639
00000100:00100000:2.0:1766099771.169272:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099771.169398:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231104
00000100:00100000:3.0:1766099771.169460:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1641
00000100:00100000:3.0:1766099771.169514:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417231104:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099771.169619:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099771.170803:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417231104:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 1329us (1536us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099771.170818:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1641
00010000:00010000:2.0:1766099771.327556:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099771.837778:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099771.837787:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099771.837791:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099771.837800:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4275
00000100:00080000:2.0:1766099771.837804:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099771.837811:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099771.837837:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007626dec1 x1851885432948800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099771.837897:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432948864:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099771.837948:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432948928:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099771.838010:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432948736:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099771.838242:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f31300c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432948672:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099771.838418:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099771.838570:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948928
00000100:00100000:3.0:1766099771.838626:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1158
00000100:00100000:3.0:1766099771.838633:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432948928:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099771.838691:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432948928:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (275us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099771.838699:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1158
00010000:00080000:3.0:1766099771.838739:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a41834df x1851885432948928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099778 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099771.838752:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a41834df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432948928:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099771.838795:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007626dec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432948800:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099771.838821:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099771.838847:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948800
00000100:00100000:3.0:1766099771.838869:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1642
00000100:00100000:3.0:1766099771.838872:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432948800:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099771.838967:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099771.839000:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432948800:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (179us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099771.839006:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1642
00000100:00080000:3.0:1766099771.839029:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099771.839034:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099771.839038:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007626dec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432948800:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099771.839319:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099771.839366:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948864
00000100:00100000:2.0:1766099771.839399:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2640
00000100:00100000:2.0:1766099771.839402:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007626dec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432948864:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099771.839443:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007626dec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432948864:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (126us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099771.839448:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2640
00000100:00100000:2.0:1766099771.839509:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432948864:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099771.839963:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099771.843263:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948736
00000100:00100000:2.0:1766099771.843359:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2641
00000100:00100000:2.0:1766099771.843367:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005834876b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432948736:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099771.843551:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005834876b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432948736:12345-0@lo:400:kworker/u8:0.0 Request processed in 183us (3588us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099771.843559:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2641
00000100:00100000:0.0:1766099771.843764:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432948736:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099771.848046:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099771.848198:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948672
00000100:00100000:0.0:1766099771.848266:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1789
00000100:00100000:0.0:1766099771.848274:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432948672:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099771.848343:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432948672:12345-0@lo:400:kworker/u8:0.0 Request processed in 69us (308us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099771.848351:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1789
00000100:00100000:1.0:1766099771.848471:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f31300c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432948672:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099772.349960:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099773.373759:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099774.397878:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099775.422707:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099776.286446:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099776.286545:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099776.286671:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231168
00000100:00100000:3.0:1766099776.286743:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1790
00000100:00100000:3.0:1766099776.286752:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417231168:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099776.286847:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417231168:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 95us (302us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099776.286856:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1790
00000100:00100000:1.0:1766099776.286981:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099776.287012:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231296
00000100:00100000:1.0:1766099776.287042:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1643
00000100:00100000:1.0:1766099776.287047:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417231296:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099776.287155:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099776.287188:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417231296:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 139us (208us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099776.287195:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1643
00000100:00100000:0.0:1766099776.288959:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231232
00000100:00100000:0.0:1766099776.289965:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2642
00000100:00100000:0.0:1766099776.289978:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ef41ef9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417231232:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099776.290102:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ef41ef9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417231232:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 125us (3678us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099776.290111:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2642
00010000:00010000:1.0:1766099776.446015:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099776.958449:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099776.958459:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099776.958463:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099776.958473:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4280
00000100:00080000:0.0:1766099776.958506:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099776.958514:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099776.958547:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005dcf37af x1851885432949120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099776.958574:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099776.958597:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a889833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432949056:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.958597:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432949248:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.958820:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099776.958858:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432948992:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.958888:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099776.958932:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949248
00000100:00100000:1.0:1766099776.958990:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1159
00000100:00100000:1.0:1766099776.958996:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432949248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.959054:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432949248:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (236us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099776.959061:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1159
00010000:00080000:1.0:1766099776.959107:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000744439a3 x1851885432949248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099783 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766099776.959121:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000744439a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432949248:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099776.959162:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099776.959184:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099776.959189:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949056
00000100:00100000:0.0:1766099776.959190:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432949184:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.959216:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2643
00000100:00100000:1.0:1766099776.959220:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000744439a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432949056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099776.959224:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099776.959247:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000744439a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432949056:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (86us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099776.959253:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a889833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432949056:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.959255:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2643
00000100:00100000:1.0:1766099776.959265:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949184
00000100:00100000:1.0:1766099776.959284:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2644
00000100:00100000:1.0:1766099776.959287:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432949184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099776.959291:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432948992
00000100:00100000:0.0:1766099776.959314:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1791
00000100:00100000:1.0:1766099776.959317:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008358d193 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432949184:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (92us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099776.959318:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432948992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099776.959323:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2644
00000100:00100000:1.0:1766099776.959345:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432949184:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099776.959348:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432948992:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (460us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099776.959353:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1791
00000100:00100000:1.0:1766099776.959363:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432948992:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099776.959663:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432949120:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099776.959697:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099776.959791:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949120
00000100:00100000:1.0:1766099776.959822:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1644
00000100:00100000:1.0:1766099776.959826:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067440601 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432949120:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766099776.959922:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099776.959967:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067440601 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432949120:12345-0@lo:8:kworker/u8:2.0 Request processed in 139us (269us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099776.959974:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1644
00000100:00080000:3.0:1766099776.960351:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099776.960357:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099776.960361:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432949120:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766099777.475722:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099778.493805:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099779.519083:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099780.541764:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099781.406687:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099781.406748:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099781.406820:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099781.408726:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231360
00000100:00100000:0.0:1766099781.408825:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1792
00000100:00100000:0.0:1766099781.408833:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417231360:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099781.408951:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417231360:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 117us (2132us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099781.408959:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1792
00000100:00100000:1.0:1766099781.409932:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231488
00000100:00100000:1.0:1766099781.409994:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1645
00000100:00100000:1.0:1766099781.410001:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417231488:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099781.410119:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099781.410186:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417231488:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 184us (3510us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099781.410194:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1645
00000100:00100000:3.0:1766099781.411515:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231424
00000100:00100000:3.0:1766099781.411589:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2645
00000100:00100000:3.0:1766099781.411595:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417231424:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099781.411676:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a41834df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417231424:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 81us (4929us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099781.411683:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2645
00010000:00010000:1.0:1766099781.567062:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099782.078790:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432949312:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766099782.078805:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099782.078811:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099782.078815:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099782.078820:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4285
00000100:00080000:1.0:1766099782.078822:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099782.078829:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099782.078854:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c2b7fd72 x1851885432949440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099782.078888:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099782.078906:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432949440:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099782.078920:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099782.078924:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432949568:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099782.078941:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099782.078979:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949312
00000100:00100000:0.0:1766099782.079019:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432949376:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099782.079029:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1793
00000100:00100000:3.0:1766099782.079036:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432949312:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099782.079715:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099782.079718:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432949312:12345-0@lo:400:kworker/u8:2.0 Request processed in 682us (832us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099782.079727:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1793
00000100:00100000:0.0:1766099782.079736:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099782.079742:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432949504:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099782.079764:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099782.079793:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949376
00000100:00100000:3.0:1766099782.079805:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949440
00000100:00100000:0.0:1766099782.079821:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949504
00000100:00100000:3.0:1766099782.079823:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1646
00000100:00100000:3.0:1766099782.079827:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432949440:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099782.079837:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2646
00000100:00100000:0.0:1766099782.079840:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432949376:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099782.079874:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a889833 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432949376:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (161us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099782.079880:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2646
00000100:00100000:0.0:1766099782.079893:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2647
00000100:00100000:0.0:1766099782.079896:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432949504:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099782.079916:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5132164 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432949504:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (152us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099782.079921:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2647
00010000:02020000:3.0:1766099782.079927:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099782.079957:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092fe0f9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432949504:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099782.079957:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432949440:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (1016us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099782.079963:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1646
00000100:00100000:0.0:1766099782.079966:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432949376:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099782.080781:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766099782.080790:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099782.080795:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099782.080799:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432949440:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766099782.080808:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432949312:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099782.086159:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949568
00000100:00100000:3.0:1766099782.086251:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1160
00000100:00100000:3.0:1766099782.086259:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bedb856 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432949568:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099782.086398:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bedb856 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432949568:12345-0@lo:400:kworker/u8:2.0 Request processed in 131us (5611us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099782.086407:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1160
00010000:00080000:2.0:1766099782.086530:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000070c173f7 x1851885432949568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099789 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099782.086561:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432949568:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099782.592381:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099783.619066:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099784.639151:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099785.661720:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099786.528534:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099786.528587:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099786.528635:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099786.528879:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231552
00000100:00100000:2.0:1766099786.528951:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1794
00000100:00100000:2.0:1766099786.528960:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417231552:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099786.529050:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417231552:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (580us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099786.529058:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1794
00000100:00100000:3.0:1766099786.529146:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231616
00000100:00100000:3.0:1766099786.529192:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2648
00000100:00100000:3.0:1766099786.529197:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417231616:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099786.529233:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a6cc55b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417231616:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 35us (646us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099786.529239:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2648
00000100:00100000:1.0:1766099786.529464:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231680
00000100:00100000:1.0:1766099786.529527:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1647
00000100:00100000:1.0:1766099786.529532:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417231680:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099786.529631:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099786.529664:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417231680:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 132us (1029us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099786.529675:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1647
00010000:00010000:2.0:1766099786.685927:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099787.201524:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099787.201534:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099787.201539:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099787.201549:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4290
00000100:00080000:0.0:1766099787.201553:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099787.201560:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099787.201587:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b602d77c x1851885432949760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099787.201639:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432949696:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099787.201720:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099787.201753:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099787.201758:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432949824:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099787.201780:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099787.201819:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949696
00000100:00100000:0.0:1766099787.201864:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949824
00000100:00100000:0.0:1766099787.201885:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2649
00000100:00100000:0.0:1766099787.201891:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ccf9eed1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432949696:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099787.201944:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ccf9eed1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432949696:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (225us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099787.201952:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2649
00000100:00100000:0.0:1766099787.201972:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2650
00000100:00100000:0.0:1766099787.201975:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8b9cdf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432949824:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099787.202004:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8b9cdf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432949824:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (224us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099787.202010:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2650
00000100:00100000:0.0:1766099787.202058:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432949824:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099787.202068:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041477718 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432949696:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099787.202151:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093ed14f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432949888:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099787.202192:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099787.202220:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099787.202226:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432949632:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099787.202260:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099787.202283:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432949760:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099787.202307:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099787.202457:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949888
00000100:00100000:0.0:1766099787.202520:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949760
00000100:00100000:0.0:1766099787.202537:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1161
00000100:00100000:0.0:1766099787.202541:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432949888:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099787.202810:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1648
00000100:00100000:2.0:1766099787.202815:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432949760:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766099787.202926:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099787.202967:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432949760:12345-0@lo:8:kworker/u8:0.0 Request processed in 151us (660us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099787.202973:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1648
00000100:00100000:2.0:1766099787.203217:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949632
00000100:00100000:2.0:1766099787.203246:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1795
00000100:00100000:2.0:1766099787.203250:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432949632:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099787.203282:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432949632:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (1022us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099787.203288:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1795
00000100:00100000:1.0:1766099787.203404:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce45f83d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432949632:0@lo:400:kworker/u8:0.0
00010000:00080000:1.0:1766099787.203417:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000093ed14f9 x1851885432949888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099794 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099787.203427:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093ed14f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432949888:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766099787.203470:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099787.203474:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099787.203495:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432949760:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099787.203739:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432949888:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (392us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099787.203747:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1161
00010000:00010000:1.0:1766099787.709746:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099788.733950:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099789.757782:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099790.782256:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099791.644966:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099791.645025:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099791.645131:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099791.645253:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231808
00000100:00100000:1.0:1766099791.645306:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231872
00000100:00100000:2.0:1766099791.645333:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2651
00000100:00100000:2.0:1766099791.645341:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e7c243d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417231808:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099791.645342:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1649
00000100:00100000:1.0:1766099791.645346:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417231872:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:2.0:1766099791.645431:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e7c243d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417231808:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 90us (406us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099791.645439:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2651
00010000:02020000:1.0:1766099791.645442:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099791.645507:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417231872:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 128us (344us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099791.645517:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1649
00000100:00100000:0.0:1766099791.645637:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231744
00000100:00100000:0.0:1766099791.645677:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1796
00000100:00100000:0.0:1766099791.645682:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417231744:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099791.645731:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417231744:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 49us (777us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099791.645738:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1796
00010000:00010000:1.0:1766099791.805899:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099792.317999:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099792.318007:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099792.318012:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099792.318022:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4295
00000100:00080000:1.0:1766099792.318026:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099792.318034:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099792.318062:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000070c173f7 x1851885432950080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099792.318116:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bc02f61 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950016:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099792.318198:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432949952:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099792.318211:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099792.318248:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099792.318256:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432950144:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099792.318279:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099792.318311:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432950080:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099792.318372:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099792.318403:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099792.318426:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950016
00000100:00100000:0.0:1766099792.318444:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099792.318455:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432950208:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099792.318518:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950144
00000100:00100000:0.0:1766099792.318525:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099792.318557:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2652
00000100:00100000:1.0:1766099792.318566:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092fe0f9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099792.318604:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2653
00000100:00100000:0.0:1766099792.318614:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b735f288 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432950144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099792.318635:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092fe0f9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950016:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (432us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099792.318645:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2652
00000100:00100000:0.0:1766099792.318688:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b735f288 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432950144:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (406us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099792.318697:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2653
00000100:00100000:1.0:1766099792.318711:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432950144:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099792.318730:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bc02f61 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950016:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766099792.318748:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950080
00000100:00100000:0.0:1766099792.318791:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950208
00000100:00100000:1.0:1766099792.318832:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1162
00000100:00100000:2.0:1766099792.318894:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432949952
00000100:00100000:2.0:1766099792.318929:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1797
00000100:00100000:2.0:1766099792.318933:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432949952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099792.318979:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432949952:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (581us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099792.318985:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1797
00000100:00100000:0.0:1766099792.320578:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1650
00000100:00100000:0.0:1766099792.320589:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432950080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766099792.320685:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099792.320769:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432950080:12345-0@lo:8:kworker/u8:2.0 Request processed in 178us (2394us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099792.320780:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1650
00000100:00100000:2.0:1766099792.320882:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432949952:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099792.321185:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099792.321191:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099792.321197:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432950080:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099792.325452:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432950208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099792.325635:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432950208:12345-0@lo:400:kworker/u8:2.0 Request processed in 199us (7113us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099792.325648:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1162
00010000:00080000:2.0:1766099792.327976:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000505a1a88 x1851885432950208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099799 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766099792.328001:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432950208:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766099792.829767:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099793.855873:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099794.878015:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099795.903428:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099796.764515:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099796.764579:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099796.764672:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417231936
00000100:00100000:0.0:1766099796.764739:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1798
00000100:00100000:0.0:1766099796.764749:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417231936:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099796.764844:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417231936:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 96us (265us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099796.764853:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1798
00000100:00100000:3.0:1766099796.765587:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232000
00000100:00100000:3.0:1766099796.765640:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2654
00000100:00100000:3.0:1766099796.765645:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417232000:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099796.766103:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000daafc58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417232000:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 456us (1643us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099796.766113:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2654
00000100:00100000:2.0:1766099796.771784:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099796.777403:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232064
00000100:00100000:3.0:1766099796.777510:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1651
00000100:00100000:3.0:1766099796.777519:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417232064:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099796.777626:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099796.777740:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417232064:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 220us (5967us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099796.777750:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1651
00010000:00010000:1.0:1766099796.925754:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099797.440342:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099797.440351:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099797.440355:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099797.440367:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4300
00000100:00100000:3.0:1766099797.440368:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048d710c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432950272:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766099797.440373:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099797.440383:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099797.440409:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000dc100191 x1851885432950400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099797.440467:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099797.440523:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099797.440529:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000074e0fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432950528:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099797.440561:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099797.440582:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc100191 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432950400:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766099797.440616:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099797.440949:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950336:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099797.440974:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950528
00000100:00100000:1.0:1766099797.440989:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099797.441023:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950400
00000100:00100000:1.0:1766099797.441024:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099797.441032:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432950464:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099797.441047:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1163
00000100:00100000:3.0:1766099797.441055:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb18fa0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432950528:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099797.441061:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099797.441111:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1652
00000100:00100000:1.0:1766099797.441117:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432950400:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766099797.441126:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb18fa0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432950528:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (563us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099797.441135:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1163
00010000:00080000:3.0:1766099797.441177:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000074e0fdd x1851885432950528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099804 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766099797.441190:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000074e0fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432950528:0@lo:400:kworker/u8:2.0
00010000:02020000:1.0:1766099797.441224:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099797.441253:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950336
00000100:00100000:1.0:1766099797.441265:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432950400:12345-0@lo:8:kworker/u8:2.0 Request processed in 147us (648us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099797.441273:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1652
00000100:00100000:3.0:1766099797.441294:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950464
00000100:00100000:1.0:1766099797.441310:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2655
00000100:00100000:1.0:1766099797.441316:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950336:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766099797.441322:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2656
00000100:00100000:3.0:1766099797.441327:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432950464:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099797.441361:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157c8bb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950336:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (373us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099797.441372:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f110b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432950464:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (313us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099797.441374:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2655
00000100:00100000:3.0:1766099797.441384:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2656
00000100:00080000:1.0:1766099797.441416:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099797.441421:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099797.441426:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc100191 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432950400:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766099797.441450:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432950464:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766099797.441458:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950336:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099797.441609:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950272
00000100:00100000:2.0:1766099797.441652:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1799
00000100:00100000:2.0:1766099797.441657:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432950272:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766099797.441701:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432950272:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (1237us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099797.441709:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1799
00000100:00100000:2.0:1766099797.441749:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048d710c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432950272:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766099797.952341:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099798.974366:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099799.998213:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099801.025812:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099801.886574:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099801.886629:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099801.886746:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232128
00000100:00100000:0.0:1766099801.886805:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1800
00000100:00100000:0.0:1766099801.886812:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417232128:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099801.886886:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417232128:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 73us (319us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099801.886893:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1800
00000100:00100000:2.0:1766099801.887091:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232192
00000100:00100000:2.0:1766099801.887134:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2657
00000100:00100000:2.0:1766099801.887138:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417232192:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099801.887167:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7d0a8c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417232192:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 28us (539us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099801.887174:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2657
00000100:00100000:2.0:1766099801.891102:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099801.891204:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232256
00000100:00100000:0.0:1766099801.891258:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1653
00000100:00100000:0.0:1766099801.891265:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417232256:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099801.891373:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099801.892267:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417232256:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 998us (1289us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099801.892280:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1653
00010000:00010000:2.0:1766099802.045716:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099802.557802:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099802.557812:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099802.557815:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099802.557824:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4305
00000100:00080000:1.0:1766099802.557828:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099802.557834:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099802.557858:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007ed1b667 x1851885432950720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099802.557918:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007de97d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950656:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099802.557919:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432950784:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099802.558005:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099802.558030:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432950848:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099802.558069:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ed1b667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432950720:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099802.558078:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099802.558121:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099802.558123:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099802.558133:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432950592:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099802.558183:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099802.558187:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950848
00000100:00100000:0.0:1766099802.558228:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950720
00000100:00100000:1.0:1766099802.558267:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1164
00000100:00100000:1.0:1766099802.558274:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432950848:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099802.558312:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1654
00000100:00100000:0.0:1766099802.558319:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c45bca0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432950720:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099802.558347:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432950848:12345-0@lo:400:kworker/u8:0.0 Request processed in 70us (276us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099802.558356:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1164
00000100:00100000:1.0:1766099802.558387:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:02020000:0.0:1766099802.558416:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00010000:00080000:1.0:1766099802.558441:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000be044d66 x1851885432950848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099809 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099802.558457:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432950848:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099802.558519:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c45bca0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432950720:12345-0@lo:8:kworker/u8:0.0 Request processed in 196us (398us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099802.558529:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1654
00000100:00080000:1.0:1766099802.558556:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099802.558561:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099802.558566:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ed1b667 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432950720:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099802.558609:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950592
00000100:00100000:1.0:1766099802.558622:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950784
00000100:00100000:0.0:1766099802.558652:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1801
00000100:00100000:0.0:1766099802.558660:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432950592:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099802.558668:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2658
00000100:00100000:1.0:1766099802.558675:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e67a7300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432950784:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099802.559265:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432950592:12345-0@lo:400:kworker/u8:0.0 Request processed in 602us (1077us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099802.559279:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1801
00000100:00100000:1.0:1766099802.559280:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e67a7300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432950784:12345-0@lo:400:kworker/u8:0.0 Request processed in 599us (1271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099802.559303:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2658
00000100:00100000:1.0:1766099802.559326:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950656
00000100:00100000:0.0:1766099802.559360:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432950592:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099802.559362:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2659
00000100:00100000:1.0:1766099802.559371:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016c31705 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950656:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099802.559452:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432950784:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099802.559594:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016c31705 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950656:12345-0@lo:400:kworker/u8:0.0 Request processed in 221us (1205us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099802.559602:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2659
00000100:00100000:1.0:1766099802.559641:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007de97d21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950656:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099803.074861:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099804.093809:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099805.118073:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099806.141768:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099807.007743:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099807.007801:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099807.007834:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232320
00000100:00100000:0.0:1766099807.007905:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1802
00000100:00100000:0.0:1766099807.007913:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232320:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099807.008006:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232320:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 93us (278us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099807.008013:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1802
00000100:00100000:3.0:1766099807.008249:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232384
00000100:00100000:3.0:1766099807.008299:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2660
00000100:00100000:3.0:1766099807.008306:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417232384:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099807.008348:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007de97d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417232384:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 42us (547us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099807.008355:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2660
00000100:00100000:1.0:1766099807.012354:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099807.012470:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232448
00000100:00100000:3.0:1766099807.012554:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1655
00000100:00100000:3.0:1766099807.012561:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417232448:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099807.012664:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099807.012749:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417232448:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 187us (406us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099807.012758:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1655
00010000:00010000:1.0:1766099807.168929:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099807.677760:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099807.677770:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099807.677775:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099807.677785:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4311
00000100:00080000:1.0:1766099807.677789:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099807.677796:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099807.677825:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c3a4d1db x1851885432951040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099807.677899:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432951040:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099807.677988:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099807.678060:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951040
00000100:00100000:1.0:1766099807.678098:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1656
00000100:00100000:1.0:1766099807.678104:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432951040:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099807.678214:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099807.678255:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432951040:12345-0@lo:8:kworker/u8:0.0 Request processed in 150us (268us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099807.678262:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1656
00000100:00080000:1.0:1766099807.678300:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099807.678303:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099807.678307:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432951040:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099807.678770:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950976:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.678811:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099807.678844:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099807.678849:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432951104:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.678867:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099807.678898:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950976
00000100:00100000:3.0:1766099807.678930:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951104
00000100:00100000:3.0:1766099807.678944:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2661
00000100:00100000:3.0:1766099807.678948:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950976:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.679000:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b30508 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432950976:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (189us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099807.679005:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2661
00000100:00100000:3.0:1766099807.679017:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2662
00000100:00100000:3.0:1766099807.679020:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a3dc47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432951104:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.679039:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a3dc47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432951104:12345-0@lo:400:kworker/u8:0.0 Request processed in 18us (172us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099807.679044:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2662
00000100:00100000:3.0:1766099807.679070:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000505a1a88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432951104:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.679089:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1197d22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432950976:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099807.679217:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432951168:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099807.679253:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099807.679271:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099807.679277:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432950912:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099807.679301:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099807.679327:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432950912
00000100:00100000:2.0:1766099807.679354:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1803
00000100:00100000:2.0:1766099807.679358:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432950912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099807.679391:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432950912:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (89us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099807.679397:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1803
00000100:00100000:2.0:1766099807.679428:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be044d66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432950912:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.679620:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951168
00000100:00100000:3.0:1766099807.679654:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1165
00000100:00100000:3.0:1766099807.679658:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432951168:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099807.679686:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432951168:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (434us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099807.679691:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1165
00010000:00080000:2.0:1766099807.679846:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000070c173f7 x1851885432951168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099814 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766099807.679856:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432951168:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766099808.189945:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099809.216139:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099810.238317:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099811.264054:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099812.127700:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099812.127782:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099812.127839:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099812.128003:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232512
00000100:00100000:0.0:1766099812.128029:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232576
00000100:00100000:0.0:1766099812.128083:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2663
00000100:00100000:3.0:1766099812.128091:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1804
00000100:00100000:0.0:1766099812.128096:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417232576:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099812.128100:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232512:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.128210:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bc02f61 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417232576:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 116us (523us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099812.128219:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2663
00000100:00100000:2.0:1766099812.128310:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232640
00000100:00100000:2.0:1766099812.128344:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1657
00000100:00100000:2.0:1766099812.128349:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417232640:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:3.0:1766099812.128447:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232512:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 348us (607us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099812.128455:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1804
00010000:02020000:2.0:1766099812.128460:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099812.128530:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417232640:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 179us (747us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099812.128537:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1657
00010000:00010000:1.0:1766099812.285775:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099812.800193:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099812.800203:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099812.800208:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099812.800218:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4316
00000100:00080000:0.0:1766099812.800222:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099812.800230:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099812.800258:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002eb4c88b x1851885432951360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099812.800327:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432951424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.800414:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099812.800456:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099812.800462:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432951296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.800503:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099812.800546:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951424
00000100:00100000:0.0:1766099812.800596:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951296
00000100:00100000:0.0:1766099812.800617:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2664
00000100:00100000:0.0:1766099812.800637:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2cdda05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432951424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.800695:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2cdda05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432951424:12345-0@lo:400:kworker/u8:3.0 Request processed in 70us (282us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099812.800721:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2664
00000100:00100000:0.0:1766099812.800745:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2665
00000100:00100000:0.0:1766099812.800750:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be292ac7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432951296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.800785:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be292ac7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432951296:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (282us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099812.800790:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2665
00000100:00100000:0.0:1766099812.800823:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432951360:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099812.800852:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099812.800883:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951360
00000100:00100000:0.0:1766099812.800901:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1658
00000100:00100000:0.0:1766099812.800905:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432951360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099812.801002:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099812.801040:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432951360:12345-0@lo:8:kworker/u8:3.0 Request processed in 134us (187us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099812.801046:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1658
00000100:00100000:0.0:1766099812.801085:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432951296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.801095:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432951424:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766099812.801126:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099812.801130:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099812.801133:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432951360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099812.801253:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432951232:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099812.801294:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099812.801330:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099812.801336:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432951488:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099812.801363:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099812.801389:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951488
00000100:00100000:2.0:1766099812.801415:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1166
00000100:00100000:2.0:1766099812.801419:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432951488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099812.801452:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432951488:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (88us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099812.801457:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1166
00010000:00080000:2.0:1766099812.801514:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006ca3c93d x1851885432951488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099819 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099812.801524:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432951488:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.801694:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951232
00000100:00100000:0.0:1766099812.801733:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1805
00000100:00100000:0.0:1766099812.801737:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432951232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099812.801770:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432951232:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (476us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099812.801780:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1805
00000100:00100000:2.0:1766099812.801858:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432951232:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099813.309793:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099814.333751:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099815.357847:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099816.381820:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099817.245393:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099817.245463:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099817.245523:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232832
00000100:00100000:1.0:1766099817.245567:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099817.245583:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1659
00000100:00100000:0.0:1766099817.245593:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417232832:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:1.0:1766099817.245604:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232768
00000100:00100000:1.0:1766099817.245650:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2666
00000100:00100000:1.0:1766099817.245655:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417232768:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00010000:02020000:0.0:1766099817.245701:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099817.245779:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417232832:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 187us (399us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099817.245788:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1659
00000100:00100000:1.0:1766099817.245954:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000505a1a88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417232768:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 297us (387us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099817.245963:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2666
00000100:00100000:3.0:1766099817.246057:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232704
00000100:00100000:3.0:1766099817.246091:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1806
00000100:00100000:3.0:1766099817.246095:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232704:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099817.246143:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232704:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 46us (682us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099817.246149:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1806
00010000:00010000:2.0:1766099817.407659:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099817.921439:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099817.921446:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099817.921449:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099817.921457:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4321
00000100:00080000:2.0:1766099817.921460:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099817.921466:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099817.921506:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000394ecbcb x1851885432951680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099817.921567:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432951616:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099817.921629:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099817.921660:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099817.921665:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cb308ba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432951744:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099817.921682:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099817.921721:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951616
00000100:00100000:2.0:1766099817.921753:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951744
00000100:00100000:2.0:1766099817.921767:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2667
00000100:00100000:2.0:1766099817.921772:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432951616:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099817.921812:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed576acc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432951616:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (184us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099817.921818:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2667
00000100:00100000:2.0:1766099817.921834:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2668
00000100:00100000:2.0:1766099817.921837:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432951744:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099817.921853:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004039bd81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432951744:12345-0@lo:400:kworker/u8:3.0 Request processed in 16us (172us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099817.921858:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2668
00000100:00100000:2.0:1766099817.921884:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cb308ba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432951744:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099817.921892:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed5b60a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432951616:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099817.922552:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ad50fe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432951552:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099817.922589:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099817.922628:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099817.922633:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d271259f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432951808:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099817.922658:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099817.922897:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000394ecbcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432951680:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099817.922926:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099817.923021:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951808
00000100:00100000:1.0:1766099817.923056:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951680
00000100:00100000:1.0:1766099817.923075:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1167
00000100:00100000:1.0:1766099817.923079:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432951808:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099817.923120:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432951808:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (461us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099817.923126:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1167
00000100:00100000:1.0:1766099817.923150:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1660
00000100:00100000:1.0:1766099817.923153:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432951680:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099817.923243:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099817.923269:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432951680:12345-0@lo:8:kworker/u8:3.0 Request processed in 115us (343us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099817.923275:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1660
00000100:00080000:0.0:1766099817.923559:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099817.923564:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099817.923568:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000394ecbcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432951680:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099817.923978:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951552
00000100:00100000:2.0:1766099817.924023:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1807
00000100:00100000:2.0:1766099817.924027:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432951552:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099817.924065:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432951552:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (1476us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099817.924074:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1807
00010000:00080000:3.0:1766099817.924236:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d271259f x1851885432951808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099824 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099817.924246:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d271259f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432951808:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099817.924253:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ad50fe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432951552:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099818.429796:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099819.453864:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099820.477934:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099821.502156:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099822.367231:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099822.367276:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099822.367309:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099822.367466:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232960
00000100:00100000:2.0:1766099822.367545:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2669
00000100:00100000:2.0:1766099822.367551:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be044d66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417232960:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099822.367631:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be044d66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417232960:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (354us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099822.367639:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2669
00000100:00100000:2.0:1766099822.367662:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233024
00000100:00100000:2.0:1766099822.367681:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1661
00000100:00100000:2.0:1766099822.367685:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1197d22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417233024:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099822.367777:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099822.367805:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1197d22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417233024:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 118us (496us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099822.367811:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1661
00000100:00100000:0.0:1766099822.368878:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417232896
00000100:00100000:0.0:1766099822.368924:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1808
00000100:00100000:0.0:1766099822.368929:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232896:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099822.368981:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417232896:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 51us (1757us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099822.368988:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1808
00010000:00010000:1.0:1766099822.528978:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099823.038805:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099823.038812:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:0.0:1766099823.038814:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432951872:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766099823.038815:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099823.038820:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4326
00000100:00080000:1.0:1766099823.038822:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099823.038826:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099823.038845:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c2deff23 x1851885432952000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099823.038884:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099823.038894:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000949bc664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432952064:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099823.038923:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099823.038928:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de0eb61b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432952128:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099823.038937:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099823.038953:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099823.038955:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099823.038959:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432951936:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099823.038973:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2deff23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952000:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099823.038977:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099823.038990:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099823.039020:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951872
00000100:00100000:0.0:1766099823.039057:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1809
00000100:00100000:0.0:1766099823.039062:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432951872:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099823.039063:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952064
00000100:00100000:1.0:1766099823.039086:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432951936
00000100:00100000:0.0:1766099823.039114:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432951872:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (231us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099823.039115:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2670
00000100:00100000:1.0:1766099823.039119:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014d07368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432952064:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099823.039122:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1809
00000100:00100000:1.0:1766099823.039150:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014d07368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432952064:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (213us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099823.039152:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952128
00000100:00100000:1.0:1766099823.039156:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2670
00000100:00100000:1.0:1766099823.039166:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2671
00000100:00100000:1.0:1766099823.039169:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007baf6919 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432951936:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099823.039171:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952000
00000100:00100000:0.0:1766099823.039185:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1168
00000100:00100000:0.0:1766099823.039188:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432952128:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099823.039190:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007baf6919 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432951936:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (213us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099823.039195:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2671
00000100:00100000:0.0:1766099823.039224:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432952128:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099823.039228:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432951936:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099823.039230:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1168
00000100:00100000:1.0:1766099823.039238:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000949bc664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432952064:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099823.039241:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1662
00000100:00100000:0.0:1766099823.039244:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432952000:12345-0@lo:8:kworker/u8:0.0
00010000:00080000:1.0:1766099823.039286:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000de0eb61b x1851885432952128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099830 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099823.039295:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de0eb61b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432952128:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099823.039301:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3a4d1db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432951872:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766099823.039351:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099823.039392:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432952000:12345-0@lo:8:kworker/u8:0.0 Request processed in 145us (400us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099823.039398:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1662
00000100:00080000:1.0:1766099823.041883:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099823.041893:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099823.041899:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2deff23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952000:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766099823.549799:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099824.574162:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099825.597706:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099826.621736:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099827.484397:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099827.484454:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099827.484576:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233152
00000100:00100000:1.0:1766099827.484629:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099827.484655:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2672
00000100:00100000:2.0:1766099827.484668:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de0eb61b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233152:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099827.484684:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233088
00000100:00100000:1.0:1766099827.484735:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1810
00000100:00100000:1.0:1766099827.484742:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417233088:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099827.484763:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de0eb61b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233152:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 98us (307us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099827.484772:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2672
00000100:00100000:1.0:1766099827.484808:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417233088:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 62us (177us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099827.484831:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1810
00000100:00100000:0.0:1766099827.485102:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233216
00000100:00100000:0.0:1766099827.485143:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1663
00000100:00100000:0.0:1766099827.485148:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2deff23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417233216:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099827.485251:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099827.485307:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2deff23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417233216:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 158us (921us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099827.485315:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1663
00010000:00010000:1.0:1766099827.645666:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099828.159266:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432952192:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099828.159339:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099828.159959:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952192
00000100:00100000:1.0:1766099828.160002:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1811
00000100:00100000:1.0:1766099828.160008:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432952192:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099828.160281:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432952192:12345-0@lo:400:kworker/u8:3.0 Request processed in 273us (943us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099828.160289:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1811
00000100:00100000:0.0:1766099828.160364:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432952192:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099828.160736:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432952256:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099828.160767:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099828.161559:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952256
00000100:00100000:0.0:1766099828.161588:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2673
00000100:00100000:0.0:1766099828.161593:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432952256:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099828.161630:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057a445fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432952256:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (863us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099828.161637:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2673
00000100:00100000:0.0:1766099828.161665:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432952256:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099828.161728:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099828.161733:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099828.161736:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099828.161741:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4331
00000100:00080000:3.0:1766099828.161743:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766099828.161747:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099828.161766:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000001d84c111 x1851885432952320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099828.162017:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432952448:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099828.162045:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099828.162091:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952320:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099828.162115:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099828.162142:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952448
00000100:00100000:1.0:1766099828.162165:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952320
00000100:00100000:1.0:1766099828.162180:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1169
00000100:00100000:1.0:1766099828.162183:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432952448:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099828.162212:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432952448:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (167us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099828.162217:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1169
00000100:00100000:1.0:1766099828.162229:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1664
00000100:00100000:1.0:1766099828.162232:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7b8d346 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432952320:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099828.162323:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099828.162350:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7b8d346 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432952320:12345-0@lo:8:kworker/u8:3.0 Request processed in 117us (235us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099828.162356:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1664
00000100:00080000:1.0:1766099828.162472:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099828.162490:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099828.162494:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d84c111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952320:0@lo:8:kworker/u8:3.0
00010000:00080000:1.0:1766099828.162514:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000024346c91 x1851885432952448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099835 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099828.162522:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024346c91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432952448:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099828.162581:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432952384:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099828.162616:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099828.165994:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952384
00000100:00100000:2.0:1766099828.166071:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2674
00000100:00100000:2.0:1766099828.166076:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432952384:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099828.166152:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5a72339 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432952384:12345-0@lo:400:kworker/u8:3.0 Request processed in 76us (3536us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099828.166159:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2674
00000100:00100000:2.0:1766099828.166197:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000492edb9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432952384:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099828.669709:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099829.695612:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099830.717714:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099831.741744:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099832.605186:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099832.609083:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099832.610463:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233280
00000100:00100000:2.0:1766099832.610544:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1812
00000100:00100000:2.0:1766099832.610550:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417233280:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099832.610627:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417233280:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 76us (5449us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099832.610634:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1812
00000100:00100000:1.0:1766099832.613144:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099832.613256:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233408
00000100:00100000:3.0:1766099832.613301:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1665
00000100:00100000:3.0:1766099832.613306:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417233408:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099832.613403:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099832.613682:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233344
00000100:00100000:0.0:1766099832.613738:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2675
00000100:00100000:0.0:1766099832.613743:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2deff23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233344:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099832.614171:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417233408:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 863us (1033us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099832.614180:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1665
00000100:00100000:0.0:1766099832.614451:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2deff23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233344:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 706us (5374us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099832.614462:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2675
00010000:00010000:2.0:1766099832.767130:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099833.277700:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099833.277708:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099833.277711:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099833.277719:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4336
00000100:00080000:2.0:1766099833.277722:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099833.277728:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099833.277751:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000394ecbcb x1851885432952640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099833.277810:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432952704:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099833.277895:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099833.277944:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099833.277951:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d271259f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432952576:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099833.277975:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099833.278009:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952704
00000100:00100000:2.0:1766099833.278084:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952576
00000100:00100000:2.0:1766099833.278105:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2676
00000100:00100000:2.0:1766099833.278110:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432952704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099833.278160:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ad50fe1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432952704:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (266us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099833.278167:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2676
00000100:00100000:2.0:1766099833.278184:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2677
00000100:00100000:2.0:1766099833.278187:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432952576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099833.278212:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed5b60a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432952576:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (237us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099833.278218:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2677
00000100:00100000:2.0:1766099833.278256:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d271259f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432952576:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099833.278264:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432952704:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278362:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432952768:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278395:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099833.278425:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099833.278430:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cb308ba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432952512:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278456:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099833.278508:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000394ecbcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952640:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278530:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099833.278558:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952512
00000100:00100000:1.0:1766099833.278581:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1813
00000100:00100000:1.0:1766099833.278584:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432952512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278613:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432952512:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (156us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099833.278618:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1813
00000100:00100000:1.0:1766099833.278644:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952768
00000100:00100000:1.0:1766099833.278662:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952640
00000100:00100000:1.0:1766099833.278674:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1170
00000100:00100000:1.0:1766099833.278677:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037d49a48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432952768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278704:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037d49a48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432952768:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (309us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099833.278709:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1170
00000100:00100000:1.0:1766099833.278719:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1666
00000100:00100000:1.0:1766099833.278722:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432952640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099833.278810:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099833.278840:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432952640:12345-0@lo:8:kworker/u8:3.0 Request processed in 117us (309us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099833.278846:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1666
00000100:00080000:1.0:1766099833.278874:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099833.278877:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099833.278881:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000394ecbcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952640:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099833.278901:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cb308ba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432952512:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766099833.278910:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000640ec22e x1851885432952768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099840 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099833.278938:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000640ec22e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432952768:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099833.791638:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099834.816275:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099835.837864:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099836.896805:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099837.731361:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099837.733330:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099837.733388:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099837.733729:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233600
00000100:00100000:3.0:1766099837.733795:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1667
00000100:00100000:3.0:1766099837.733805:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417233600:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:0.0:1766099837.733880:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233536
00010000:02020000:3.0:1766099837.733916:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099837.733953:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2678
00000100:00100000:0.0:1766099837.733961:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417233536:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099837.734015:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417233600:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 207us (2663us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099837.734028:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1667
00000100:00100000:0.0:1766099837.734053:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000640ec22e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417233536:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 88us (662us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099837.734064:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2678
00000100:00100000:1.0:1766099837.736285:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233472
00000100:00100000:1.0:1766099837.736372:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1814
00000100:00100000:1.0:1766099837.736380:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417233472:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099837.736464:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417233472:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 85us (3146us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099837.736472:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1814
00010000:00010000:1.0:1766099837.950460:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099838.399322:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a458df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432952832:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099838.399458:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099838.399560:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952832
00000100:00100000:1.0:1766099838.399622:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1815
00000100:00100000:1.0:1766099838.399629:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432952832:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766099838.401159:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099838.401174:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099838.401178:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099838.401185:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4341
00000100:00080000:0.0:1766099838.401189:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766099838.401196:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099838.401568:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432952832:12345-0@lo:400:kworker/u8:0.0 Request processed in 1935us (2109us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099838.401581:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1815
00000100:00100000:1.0:1766099838.401667:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d94121f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432952896:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099838.401722:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099838.401777:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952896
00000100:00100000:1.0:1766099838.401815:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2679
00000100:00100000:0.0:1766099838.401819:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005a16d6f1 x1851885432952960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099838.401820:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a90eed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432952896:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099838.401863:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a90eed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432952896:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (142us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099838.401873:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2679
00000100:00100000:0.0:1766099838.401889:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952960:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099838.401919:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d94121f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432952896:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099838.401928:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099838.401937:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099838.401942:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432953024:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099838.401963:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099838.401967:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432952960
00000100:00100000:0.0:1766099838.402007:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1668
00000100:00100000:0.0:1766099838.402011:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432952960:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766099838.402140:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099838.402183:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432952960:12345-0@lo:8:kworker/u8:0.0 Request processed in 170us (254us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099838.402198:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1668
00000100:00080000:0.0:1766099838.402239:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099838.402243:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099838.402246:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432952960:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099838.402269:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432953088:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099838.402303:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099838.402336:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953088
00000100:00100000:0.0:1766099838.402367:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1171
00000100:00100000:0.0:1766099838.402370:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432953088:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099838.402410:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432953088:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (108us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099838.402425:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1171
00010000:00080000:0.0:1766099838.402455:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005bf7662d x1851885432953088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099845 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099838.402465:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432953088:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099838.403014:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a458df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432952832:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099838.405518:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953024
00000100:00100000:1.0:1766099838.405601:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2680
00000100:00100000:1.0:1766099838.405608:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076159a69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432953024:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099838.405733:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076159a69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432953024:12345-0@lo:400:kworker/u8:0.0 Request processed in 124us (3769us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099838.405742:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2680
00000100:00100000:1.0:1766099838.405805:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432953024:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766099838.975529:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099839.998676:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099841.031271:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099842.047610:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099842.848300:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099842.850911:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233664
00000100:00100000:3.0:1766099842.851009:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1816
00000100:00100000:3.0:1766099842.851017:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417233664:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099842.852513:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417233664:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1459us (4185us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099842.852532:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1816
00000100:00100000:3.0:1766099842.856201:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099842.856370:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099842.856759:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233792
00000100:00100000:1.0:1766099842.857696:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233728
00000100:00100000:1.0:1766099842.857763:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2681
00000100:00100000:1.0:1766099842.857771:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024346c91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233728:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099842.857872:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024346c91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233728:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 101us (1503us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099842.857881:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2681
00000100:00100000:0.0:1766099842.858044:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1669
00000100:00100000:0.0:1766099842.858050:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417233792:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099842.858145:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099842.858195:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417233792:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 144us (2005us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099842.858202:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1669
00010000:00010000:1.0:1766099843.070138:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099843.519033:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432953152:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099843.519138:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099843.519235:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953152
00000100:00100000:3.0:1766099843.519393:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1817
00000100:00100000:3.0:1766099843.519401:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432953152:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099843.519533:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099843.519539:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099843.519545:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099843.519551:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4346
00000100:00080000:1.0:1766099843.519554:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766099843.519560:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099843.519584:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000061093366 x1851885432953280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099843.520182:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432953152:12345-0@lo:400:kworker/u8:3.0 Request processed in 777us (1043us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099843.520192:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1817
00000100:00100000:3.0:1766099843.520276:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432953216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099843.520512:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432953344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099843.520562:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099843.520604:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432953280:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099843.520634:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099843.520677:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953344
00000100:00100000:1.0:1766099843.520717:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2682
00000100:00100000:1.0:1766099843.520722:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432953344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099843.520773:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e27d7f39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432953344:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (210us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099843.520779:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2682
00000100:00100000:1.0:1766099843.520823:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df6e7141 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432953344:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099843.520916:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe5d403 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432953152:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099843.520929:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099843.520938:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432953408:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099843.520976:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099843.521215:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953280
00000100:00100000:3.0:1766099843.521418:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099843.521455:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953216
00000100:00100000:3.0:1766099843.521500:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2683
00000100:00100000:3.0:1766099843.521506:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d84c111 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432953216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099843.521547:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d84c111 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432953216:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (129us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099843.521553:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2683
00000100:00100000:3.0:1766099843.521582:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432953216:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099843.522809:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953408
00000100:00100000:2.0:1766099843.522889:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1172
00000100:00100000:2.0:1766099843.522896:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432953408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099843.522991:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432953408:12345-0@lo:400:kworker/u8:3.0 Request processed in 95us (2014us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099843.522999:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1172
00000100:00100000:2.0:1766099843.523021:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1670
00000100:00100000:2.0:1766099843.523024:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432953280:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099843.523121:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099843.523160:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432953280:12345-0@lo:8:kworker/u8:3.0 Request processed in 134us (2525us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099843.523167:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1670
00010000:00080000:2.0:1766099843.523211:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e3400fcc x1851885432953408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099850 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099843.523224:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3400fcc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432953408:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099843.523433:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099843.523438:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099843.523442:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432953280:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099844.093916:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099845.117789:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099846.142676:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099847.165806:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099847.967125:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099847.967190:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099847.967312:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233920
00000100:00100000:1.0:1766099847.967393:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2684
00000100:00100000:1.0:1766099847.967401:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233920:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099847.967535:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c1aab37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417233920:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 133us (421us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099847.967544:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2684
00000100:00100000:3.0:1766099847.968714:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099847.969244:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233984
00000100:00100000:0.0:1766099847.969378:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417233856
00000100:00100000:0.0:1766099847.969437:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1818
00000100:00100000:0.0:1766099847.969446:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417233856:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099847.969548:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417233856:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 102us (2359us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099847.969556:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1818
00000100:00100000:1.0:1766099847.970379:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1671
00000100:00100000:1.0:1766099847.970387:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417233984:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099847.970506:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099847.976843:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417233984:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 6440us (8120us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099847.976865:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1671
00010000:00010000:2.0:1766099848.194082:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099848.637821:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099848.637831:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099848.637836:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099848.637845:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4351
00000100:00080000:2.0:1766099848.637849:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766099848.637856:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099848.637881:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009453e80d x1851885432953600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099848.638286:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432953728:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638370:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099848.638406:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099848.638411:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000394ecbcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432953472:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638443:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099848.638473:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432953664:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638528:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099848.638548:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099848.638553:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d271259f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432953536:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638575:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099848.638598:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009453e80d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432953600:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638622:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099848.638663:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953664
00000100:00100000:1.0:1766099848.638711:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953536
00000100:00100000:1.0:1766099848.638733:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2685
00000100:00100000:1.0:1766099848.638739:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bedb856 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432953664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638798:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bedb856 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432953664:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (270us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099848.638807:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2685
00000100:00100000:1.0:1766099848.638826:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2686
00000100:00100000:1.0:1766099848.638829:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432953536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638854:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f31300c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432953536:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (278us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099848.638859:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2686
00000100:00100000:1.0:1766099848.638899:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d271259f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432953536:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638923:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432953664:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.638958:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953728
00000100:00100000:3.0:1766099848.639427:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953472
00000100:00100000:3.0:1766099848.639462:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1819
00000100:00100000:3.0:1766099848.639466:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432953472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099848.639542:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432953472:12345-0@lo:400:kworker/u8:0.0 Request processed in 74us (1098us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099848.639548:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1819
00000100:00100000:3.0:1766099848.639591:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000394ecbcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432953472:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099848.640547:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953600
00000100:00100000:2.0:1766099848.640629:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1173
00000100:00100000:2.0:1766099848.640636:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432953728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099848.640701:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432953728:12345-0@lo:400:kworker/u8:0.0 Request processed in 65us (2332us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099848.640707:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1173
00000100:00100000:2.0:1766099848.640723:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1672
00000100:00100000:2.0:1766099848.640726:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432953600:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766099848.640824:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099848.640865:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432953600:12345-0@lo:8:kworker/u8:0.0 Request processed in 138us (2242us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099848.640871:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1672
00000100:00080000:2.0:1766099848.640956:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099848.640961:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099848.640965:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009453e80d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432953600:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766099848.641456:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000065fe2a9a x1851885432953728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099855 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099848.641470:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432953728:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099849.213764:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099850.243400:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099851.263710:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099852.285765:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099853.112758:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099853.112815:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099853.112885:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099853.112988:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234112
00000100:00100000:1.0:1766099853.113065:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2687
00000100:00100000:1.0:1766099853.113073:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067440601 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417234112:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099853.113166:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067440601 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417234112:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 93us (351us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099853.113173:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2687
00000100:00100000:0.0:1766099853.113281:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234048
00000100:00100000:1.0:1766099853.113675:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234176
00000100:00100000:1.0:1766099853.113704:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1673
00000100:00100000:1.0:1766099853.113709:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417234176:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099853.113811:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099853.113852:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417234176:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 142us (968us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099853.113859:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1673
00000100:00100000:3.0:1766099853.114070:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1820
00000100:00100000:3.0:1766099853.114077:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417234048:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099853.114125:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417234048:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 48us (1378us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099853.114133:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1820
00010000:00010000:1.0:1766099853.309758:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099853.760015:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099853.761312:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432953792:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099853.761419:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099853.761608:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953792
00000100:00100000:1.0:1766099853.761677:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1821
00000100:00100000:1.0:1766099853.761685:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432953792:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099853.761753:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432953792:12345-0@lo:400:kworker/u8:0.0 Request processed in 67us (334us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099853.761760:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1821
00000100:00100000:3.0:1766099853.761878:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432953792:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766099853.765924:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099853.765937:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099853.765942:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099853.765952:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4357
00000100:00080000:0.0:1766099853.765957:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 6s
00000100:00080000:0.0:1766099853.765964:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099853.765991:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000057bb2ef x1851885432953920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099853.766060:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432953920:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099853.766150:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099853.766211:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953920
00000100:00100000:0.0:1766099853.766256:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1674
00000100:00100000:0.0:1766099853.766264:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432953920:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766099853.766374:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099853.766422:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432953920:12345-0@lo:8:kworker/u8:0.0 Request processed in 157us (273us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099853.766430:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1674
00000100:00100000:2.0:1766099853.766779:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432953984:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.766823:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099853.766848:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099853.766853:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062653d94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432953856:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.766886:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099853.766923:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953984
00000100:00100000:2.0:1766099853.766963:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432953856
00000100:00100000:2.0:1766099853.766982:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2688
00000100:00100000:2.0:1766099853.766986:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009453e80d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432953984:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.767038:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009453e80d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432953984:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (215us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099853.767045:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2688
00000100:00100000:2.0:1766099853.767058:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2689
00000100:00100000:2.0:1766099853.767061:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432953856:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.767082:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afd0d51e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432953856:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (196us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099853.767087:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2689
00000100:00100000:2.0:1766099853.767121:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099853.767130:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432954048:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.767155:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099853.767181:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954048
00000100:00100000:2.0:1766099853.767207:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1174
00000100:00100000:2.0:1766099853.767210:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f78b201d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432954048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.767243:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f78b201d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432954048:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (89us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099853.767249:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1174
00010000:00080000:2.0:1766099853.767274:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000055c81506 x1851885432954048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099860 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766099853.767286:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432954048:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.767320:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062653d94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432953856:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099853.767328:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002eb4c88b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432953984:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766099853.767403:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099853.767409:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099853.767414:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432953920:0@lo:8:kworker/u8:0.0
00010000:00010000:2.0:1766099854.333825:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099855.358315:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099856.382415:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099857.406124:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099858.204653:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099858.204702:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099858.204750:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099858.204865:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234240
00000100:00100000:1.0:1766099858.204889:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234368
00000100:00100000:0.0:1766099858.204929:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1822
00000100:00100000:0.0:1766099858.204939:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417234240:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099858.204949:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1675
00000100:00100000:1.0:1766099858.204957:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417234368:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:0.0:1766099858.205013:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417234240:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 74us (367us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099858.205022:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1822
00010000:02020000:1.0:1766099858.205082:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099858.205153:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417234368:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 193us (400us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099858.205162:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1675
00000100:00100000:2.0:1766099858.205297:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234304
00000100:00100000:2.0:1766099858.205348:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2690
00000100:00100000:2.0:1766099858.205353:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb18fa0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417234304:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099858.205406:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb18fa0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417234304:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 52us (703us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099858.205412:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2690
00010000:00010000:1.0:1766099858.429792:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099858.877827:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099858.877863:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:1.0:1766099858.877866:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432954176:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766099858.877867:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099858.877881:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4362
00000100:00080000:3.0:1766099858.877885:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 11s
00000100:00080000:3.0:1766099858.877890:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099858.877913:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000084be1e46 x1851885432954240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099858.877965:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003a912b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432954304:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099858.877977:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099858.878044:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099858.878125:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432954112:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099858.878857:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954176
00000100:00100000:1.0:1766099858.879132:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954304
00000100:00100000:1.0:1766099858.879177:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2691
00000100:00100000:1.0:1766099858.879185:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000394ecbcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432954176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099858.879900:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099858.879942:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2692
00000100:00100000:0.0:1766099858.879950:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099858.879955:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000399d91fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432954304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099858.879965:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084b59448 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432954368:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099858.880033:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099858.880047:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000399d91fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432954304:12345-0@lo:400:kworker/u8:3.0 Request processed in 93us (2004us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099858.880061:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2692
00000100:00100000:0.0:1766099858.880089:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084be1e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432954240:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099858.880123:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099858.880168:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954112
00000100:00100000:2.0:1766099858.880209:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954368
00000100:00100000:0.0:1766099858.880220:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1823
00000100:00100000:0.0:1766099858.880227:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432954112:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099858.880255:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954240
00000100:00100000:2.0:1766099858.880296:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1175
00000100:00100000:0.0:1766099858.880302:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432954112:12345-0@lo:400:kworker/u8:3.0 Request processed in 71us (408us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099858.880308:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432954368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099858.880328:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1823
00000100:00100000:2.0:1766099858.880376:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432954368:12345-0@lo:400:kworker/u8:3.0 Request processed in 72us (342us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099858.880389:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1175
00010000:00080000:0.0:1766099858.880406:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000084b59448 x1851885432954368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099865 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099858.880416:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1676
00000100:00100000:2.0:1766099858.880426:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432954240:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099858.880435:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084b59448 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432954368:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099858.880451:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e58708bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432954112:0@lo:400:kworker/u8:3.0
00010000:02020000:2.0:1766099858.880571:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099858.880626:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432954240:12345-0@lo:8:kworker/u8:3.0 Request processed in 200us (505us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099858.880634:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1676
00000100:00080000:0.0:1766099858.881133:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099858.881141:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099858.881146:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084be1e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432954240:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099858.881406:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003a912b62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432954304:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099858.881422:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000394ecbcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432954176:12345-0@lo:400:kworker/u8:3.0 Request processed in 2231us (3441us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099858.881436:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2691
00000100:00100000:0.0:1766099858.882194:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432954176:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099859.453859:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099860.477852:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099861.503190:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099862.526894:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099863.326982:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099863.327550:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099863.327690:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234496
00000100:00100000:2.0:1766099863.327760:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2693
00000100:00100000:2.0:1766099863.327768:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d271259f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417234496:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099863.327878:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d271259f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417234496:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 110us (909us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099863.327887:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2693
00000100:00100000:1.0:1766099863.335416:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099863.335524:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234560
00000100:00100000:2.0:1766099863.335590:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1677
00000100:00100000:2.0:1766099863.335598:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417234560:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099863.335702:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099863.335780:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417234560:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 182us (375us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099863.335788:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1677
00000100:00100000:0.0:1766099863.337073:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234432
00000100:00100000:0.0:1766099863.337144:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1824
00000100:00100000:0.0:1766099863.337151:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417234432:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099863.337225:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417234432:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 74us (9677us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099863.337232:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1824
00010000:00010000:2.0:1766099863.551073:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099863.997813:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432954432:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766099863.997813:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099863.997820:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099863.997824:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099863.997829:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4367
00000100:00080000:2.0:1766099863.997833:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 16s
00000100:00080000:2.0:1766099863.997842:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099863.997867:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000665bb208 x1851885432954560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099863.997911:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099863.997914:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432954496:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099863.997940:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099863.997946:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018d302a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432954688:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099863.997949:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099863.997969:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099863.997972:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099863.997973:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001cc58289 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432954624:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099863.997992:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099863.997993:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432954560:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099863.998015:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099863.998047:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954688
00000100:00100000:2.0:1766099863.998047:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954496
00000100:00100000:0.0:1766099863.998093:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954560
00000100:00100000:2.0:1766099863.998093:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954624
00000100:00100000:0.0:1766099863.998111:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1176
00000100:00100000:2.0:1766099863.998111:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2694
00000100:00100000:0.0:1766099863.998117:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432954688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099863.998117:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047eeb220 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432954496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099863.998168:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432954688:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (194us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099863.998168:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047eeb220 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432954496:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (218us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099863.998176:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1176
00000100:00100000:2.0:1766099863.998176:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2694
00000100:00100000:2.0:1766099863.998193:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2695
00000100:00100000:0.0:1766099863.998194:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1678
00000100:00100000:0.0:1766099863.998197:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d710c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432954560:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099863.998197:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000452fc292 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432954624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099863.998218:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000452fc292 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432954624:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (226us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099863.998224:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2695
00000100:00100000:2.0:1766099863.998263:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001cc58289 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432954624:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099863.998272:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065fe2a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432954496:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766099863.998296:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099863.998341:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d710c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432954560:12345-0@lo:8:kworker/u8:0.0 Request processed in 143us (327us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099863.998348:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1678
00000100:00080000:0.0:1766099863.998373:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099863.998377:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099863.998380:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000665bb208 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432954560:0@lo:8:kworker/u8:0.0
00010000:00080000:0.0:1766099863.998402:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000018d302a6 x1851885432954688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099870 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099863.998408:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954432
00000100:00100000:0.0:1766099863.998410:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018d302a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432954688:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099863.998456:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1825
00000100:00100000:3.0:1766099863.998461:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432954432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099863.998534:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432954432:12345-0@lo:400:kworker/u8:0.0 Request processed in 73us (626us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099863.998541:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1825
00000100:00100000:0.0:1766099863.998698:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003204340c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432954432:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099864.575425:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099865.597772:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099866.621844:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099867.649367:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099868.444913:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099868.444979:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099868.445025:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099868.445164:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234688
00000100:00100000:0.0:1766099868.445244:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2696
00000100:00100000:0.0:1766099868.445252:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417234688:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099868.445353:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234752
00000100:00100000:0.0:1766099868.445354:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3400fcc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417234688:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 101us (451us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099868.445362:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2696
00000100:00100000:3.0:1766099868.445381:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1679
00000100:00100000:3.0:1766099868.445385:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417234752:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099868.445519:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099868.445570:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417234752:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 183us (544us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099868.445578:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1679
00000100:00100000:2.0:1766099868.445794:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234624
00000100:00100000:2.0:1766099868.445839:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1826
00000100:00100000:2.0:1766099868.445844:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417234624:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099868.445875:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417234624:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 30us (897us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099868.445882:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1826
00010000:00010000:1.0:1766099868.669951:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099869.123288:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099869.123298:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099869.123302:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099869.123313:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4372
00000100:00080000:0.0:1766099869.123319:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099869.123327:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099869.123351:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000dc100191 x1851885432954880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099869.123410:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000074e0fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432954944:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099869.123518:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099869.123561:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766099869.123567:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432954816:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099869.123588:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099869.123616:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc100191 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432954880:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099869.123648:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099869.123689:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954944
00000100:00100000:0.0:1766099869.123743:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954816
00000100:00100000:0.0:1766099869.123782:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2697
00000100:00100000:0.0:1766099869.123790:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093ed14f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432954944:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099869.123799:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2698
00000100:00100000:2.0:1766099869.123809:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432954816:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099869.123858:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093ed14f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432954944:12345-0@lo:400:kworker/u8:0.0 Request processed in 67us (341us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099869.123867:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2697
00000100:00100000:2.0:1766099869.123871:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb4c88b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432954816:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (281us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099869.123880:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2698
00000100:00100000:0.0:1766099869.123939:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432954816:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099869.123954:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000074e0fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432954944:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099869.125079:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432954752:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099869.125149:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099869.125191:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099869.125198:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432955008:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099869.125226:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099869.125283:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954880
00000100:00100000:3.0:1766099869.125314:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955008
00000100:00100000:3.0:1766099869.125340:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1177
00000100:00100000:3.0:1766099869.125345:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432955008:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099869.125397:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432955008:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (171us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099869.125404:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1177
00000100:00100000:3.0:1766099869.125421:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680
00000100:00100000:3.0:1766099869.125425:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432954880:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099869.125558:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099869.125600:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432954880:12345-0@lo:8:kworker/u8:0.0 Request processed in 173us (1952us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099869.125608:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680
00000100:00080000:3.0:1766099869.125644:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099869.125648:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099869.125653:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc100191 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432954880:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766099869.125684:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b602d77c x1851885432955008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099876 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099869.125696:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432955008:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099869.127066:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432954752
00000100:00100000:2.0:1766099869.127147:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1827
00000100:00100000:2.0:1766099869.127154:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432954752:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099869.127262:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432954752:12345-0@lo:400:kworker/u8:0.0 Request processed in 107us (2114us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099869.127270:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1827
00000100:00100000:3.0:1766099869.127588:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432954752:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766099869.695923:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099870.721067:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099871.741835:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099872.766029:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099873.569810:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099873.569856:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099873.569927:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234880
00000100:00100000:0.0:1766099873.571654:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2699
00000100:00100000:0.0:1766099873.571666:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417234880:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099873.571770:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe5d403 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417234880:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 106us (1970us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099873.571778:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2699
00000100:00100000:1.0:1766099873.579256:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099873.579355:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234944
00000100:00100000:0.0:1766099873.579419:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1681
00000100:00100000:0.0:1766099873.579428:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417234944:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099873.579549:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099873.580136:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417234816
00000100:00100000:3.0:1766099873.580184:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1828
00000100:00100000:3.0:1766099873.580189:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417234816:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099873.580703:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417234944:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1273us (1458us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099873.580715:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1681
00000100:00100000:3.0:1766099873.580790:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417234816:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 599us (10933us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099873.580799:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1828
00010000:00010000:1.0:1766099873.790385:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099874.237905:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099874.237914:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099874.237919:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099874.237931:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4377
00000100:00080000:2.0:1766099874.237935:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099874.237941:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099874.237967:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000038aa4304 x1851885432955200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099874.238017:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000110634bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432955136:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099874.238101:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099874.238141:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099874.238146:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004004294c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432955264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099874.238166:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099874.238201:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955136
00000100:00100000:2.0:1766099874.238246:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955264
00000100:00100000:2.0:1766099874.238265:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2700
00000100:00100000:2.0:1766099874.238272:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d77d75e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432955136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099874.238326:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d77d75e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432955136:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (227us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099874.238334:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2700
00000100:00100000:2.0:1766099874.238353:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2701
00000100:00100000:2.0:1766099874.238357:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432955264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099874.238379:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000973030be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432955264:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (213us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099874.238384:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2701
00000100:00100000:2.0:1766099874.238436:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004004294c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432955264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099874.238447:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000110634bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432955136:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239258:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c91368f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432955328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239305:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099874.239332:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099874.239339:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001cc58289 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432955072:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239367:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099874.239396:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038aa4304 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432955200:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239421:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099874.239460:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955328
00000100:00100000:3.0:1766099874.239520:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955200
00000100:00100000:3.0:1766099874.239540:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1178
00000100:00100000:3.0:1766099874.239545:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de379e43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432955328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239586:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de379e43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432955328:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (281us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099874.239592:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1178
00000100:00100000:3.0:1766099874.239606:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1682
00000100:00100000:3.0:1766099874.239610:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000772d840f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432955200:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099874.239704:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099874.239749:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000772d840f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432955200:12345-0@lo:8:kworker/u8:0.0 Request processed in 137us (327us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099874.239755:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1682
00000100:00080000:3.0:1766099874.239797:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099874.239802:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099874.239806:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038aa4304 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432955200:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766099874.239832:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c91368f8 x1851885432955328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099881 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099874.239842:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c91368f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432955328:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239862:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955072
00000100:00100000:3.0:1766099874.239892:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1829
00000100:00100000:3.0:1766099874.239897:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432955072:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099874.239929:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432955072:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (562us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099874.239935:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1829
00000100:00100000:3.0:1766099874.239965:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001cc58289 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432955072:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099874.815625:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099875.837760:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099876.861752:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099877.885804:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099878.691045:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099878.691105:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099878.691165:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099878.691289:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235008
00000100:00100000:1.0:1766099878.691361:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1830
00000100:00100000:1.0:1766099878.691371:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417235008:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099878.691457:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417235008:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 86us (426us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099878.691465:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1830
00000100:00100000:3.0:1766099878.691707:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235136
00000100:00100000:3.0:1766099878.691745:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1683
00000100:00100000:3.0:1766099878.691749:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009296b030 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417235136:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099878.691845:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099878.691886:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009296b030 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417235136:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 135us (721us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099878.691892:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1683
00000100:00100000:2.0:1766099878.692084:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235072
00000100:00100000:2.0:1766099878.692132:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2702
00000100:00100000:2.0:1766099878.692137:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417235072:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099878.692195:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000074e0fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417235072:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 57us (1090us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099878.692203:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2702
00010000:00010000:2.0:1766099878.910240:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099879.357980:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432955392:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099879.358084:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099879.358557:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955392
00000100:00100000:1.0:1766099879.358608:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1831
00000100:00100000:1.0:1766099879.358615:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432955392:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099879.358694:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432955392:12345-0@lo:400:kworker/u8:3.0 Request processed in 79us (611us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099879.358703:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1831
00000100:00100000:0.0:1766099879.358716:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432955392:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099879.358855:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432955456:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099879.358894:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099879.359250:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955456
00000100:00080000:3.0:1766099879.359326:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099879.359332:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099879.359336:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099879.359341:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4382
00000100:00080000:3.0:1766099879.359343:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766099879.359347:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099879.359373:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006de96a83 x1851885432955520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099879.359424:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432955520:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099879.359460:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099879.359815:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432955584:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099879.359849:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099879.359895:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099879.359903:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084be1e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432955648:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099879.359929:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099879.359972:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955520
00000100:00100000:0.0:1766099879.359994:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955648
00000100:00100000:0.0:1766099879.360014:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1179
00000100:00100000:0.0:1766099879.360019:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432955648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099879.360062:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432955648:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (133us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099879.360069:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1179
00000100:00100000:0.0:1766099879.360086:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1684
00000100:00100000:0.0:1766099879.360090:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432955520:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099879.360229:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099879.360256:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432955520:12345-0@lo:8:kworker/u8:3.0 Request processed in 165us (796us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099879.360262:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1684
00000100:00080000:0.0:1766099879.360286:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099879.360289:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099879.360294:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432955520:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766099879.360319:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000084be1e46 x1851885432955648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099886 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099879.360330:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084be1e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432955648:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099879.363261:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955584
00000100:00100000:2.0:1766099879.363338:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2703
00000100:00100000:2.0:1766099879.363347:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000110634bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432955456:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099879.363445:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000110634bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432955456:12345-0@lo:400:kworker/u8:3.0 Request processed in 99us (4551us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099879.363453:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2703
00000100:00100000:2.0:1766099879.363473:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2704
00000100:00100000:2.0:1766099879.363498:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432955584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099879.363534:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e58708bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432955584:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (3685us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099879.363540:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2704
00000100:00100000:2.0:1766099879.363596:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432955456:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099879.363640:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432955584:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099879.933830:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099880.957919:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766099881.981833:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099883.006353:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099883.804627:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099883.805590:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235328
00000100:00100000:3.0:1766099883.805675:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1685
00000100:00100000:3.0:1766099883.805685:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417235328:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099883.805799:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099883.805907:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417235328:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 222us (1291us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099883.805917:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1685
00000100:00100000:1.0:1766099883.806214:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099883.806372:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099883.806403:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235264
00000100:00100000:1.0:1766099883.806524:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2705
00000100:00100000:1.0:1766099883.806530:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417235264:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099883.806585:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009aac6aa2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417235264:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 55us (214us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099883.806593:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2705
00000100:00100000:2.0:1766099883.808334:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235200
00000100:00100000:2.0:1766099883.809044:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1832
00000100:00100000:2.0:1766099883.809053:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417235200:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099883.809139:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417235200:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 87us (2927us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099883.809150:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1832
00010000:00010000:1.0:1766099884.029825:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099884.478212:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099884.478223:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099884.478228:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099884.478239:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4387
00000100:00080000:3.0:1766099884.478244:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766099884.478251:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099884.478283:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000096d55fda x1851885432955840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099884.478350:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432955840:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099884.478440:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099884.478537:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955840
00000100:00100000:3.0:1766099884.478582:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1686
00000100:00100000:3.0:1766099884.478590:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432955840:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099884.478708:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099884.478760:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432955840:12345-0@lo:8:kworker/u8:0.0 Request processed in 170us (322us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099884.478769:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1686
00000100:00080000:3.0:1766099884.478824:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099884.478827:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099884.478832:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096d55fda pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432955840:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099884.478945:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084b59448 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432955712:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099884.478989:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099884.479022:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099884.479027:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432955968:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099884.479050:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099884.479071:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084be1e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432955904:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099884.479098:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099884.479117:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099884.479122:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432955776:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099884.479144:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099884.479172:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955968
00000100:00100000:2.0:1766099884.479206:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1180
00000100:00100000:2.0:1766099884.479211:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432955968:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099884.479257:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432955968:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (206us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099884.479264:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1180
00010000:00080000:2.0:1766099884.479290:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006de96a83 x1851885432955968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099891 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766099884.479302:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006de96a83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432955968:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099884.480289:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955904
00000100:00100000:1.0:1766099884.480337:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955776
00000100:00100000:1.0:1766099884.480359:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2706
00000100:00100000:1.0:1766099884.480364:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432955904:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099884.480439:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9d53748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432955904:12345-0@lo:400:kworker/u8:0.0 Request processed in 74us (1340us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099884.480447:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2706
00000100:00100000:1.0:1766099884.480464:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2707
00000100:00100000:1.0:1766099884.480467:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001cc58289 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432955776:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099884.480515:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001cc58289 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432955776:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (1370us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099884.480521:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2707
00000100:00100000:1.0:1766099884.480564:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a6a67406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432955776:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099884.480576:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084be1e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432955904:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099884.480783:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432955712
00000100:00100000:3.0:1766099884.480830:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1833
00000100:00100000:3.0:1766099884.480835:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432955712:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099884.480878:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432955712:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (1890us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099884.480888:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1833
00000100:00100000:2.0:1766099884.481020:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084b59448 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432955712:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766099885.057223:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099886.077683:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099887.103892:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099888.125712:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099888.929246:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099888.931412:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235456
00000100:00100000:3.0:1766099888.931528:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2708
00000100:00100000:3.0:1766099888.931536:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417235456:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099888.931646:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026329de0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417235456:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 109us (2446us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099888.931654:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2708
00000100:00100000:1.0:1766099888.932407:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099888.932458:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099888.932565:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235392
00000100:00100000:0.0:1766099888.933587:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1834
00000100:00100000:0.0:1766099888.933598:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417235392:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099888.933701:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417235392:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 103us (1298us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099888.933710:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1834
00000100:00100000:3.0:1766099888.941146:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235520
00000100:00100000:3.0:1766099888.941241:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1687
00000100:00100000:3.0:1766099888.941250:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417235520:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099888.941355:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099888.941471:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417235520:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 221us (9012us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099888.941518:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1687
00010000:00010000:0.0:1766099889.149744:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099889.599394:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099889.599405:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099889.599409:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099889.599419:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4392
00000100:00080000:1.0:1766099889.599424:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099889.599430:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099889.599458:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000069d150bd x1851885432956160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099889.599535:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432956224:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099889.599624:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099889.599663:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099889.599669:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432956096:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099889.599692:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099889.599873:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956224
00000100:00100000:0.0:1766099889.599922:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956096
00000100:00100000:0.0:1766099889.599942:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2709
00000100:00100000:0.0:1766099889.599949:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432956224:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099889.600009:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081620938 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432956224:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (385us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099889.600017:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2709
00000100:00100000:0.0:1766099889.600038:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2710
00000100:00100000:0.0:1766099889.600041:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c45bca0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432956096:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099889.600066:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c45bca0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432956096:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (373us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099889.600072:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2710
00000100:00100000:0.0:1766099889.600094:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432956032:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099889.600128:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099889.600146:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099889.600151:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432956288:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099889.600177:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099889.600365:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2b7fd72 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432956096:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099889.600377:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000778ea5b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432956224:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099889.600521:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432956160:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099889.600558:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099889.600665:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956288
00000100:00100000:1.0:1766099889.600704:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956160
00000100:00100000:1.0:1766099889.600727:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1181
00000100:00100000:1.0:1766099889.600732:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432956288:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099889.600774:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432956288:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (597us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099889.600780:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1181
00000100:00100000:1.0:1766099889.600794:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1688
00000100:00100000:1.0:1766099889.600797:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432956160:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099889.600901:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099889.600933:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432956160:12345-0@lo:8:kworker/u8:0.0 Request processed in 134us (375us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099889.600939:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1688
00010000:00080000:1.0:1766099889.600970:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000070c173f7 x1851885432956288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099896 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099889.600982:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432956288:0@lo:400:kworker/u8:0.0
00000100:00080000:3.0:1766099889.601063:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099889.601068:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099889.601072:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432956160:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099889.601171:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956032
00000100:00100000:2.0:1766099889.601213:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1835
00000100:00100000:2.0:1766099889.601218:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432956032:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099889.601263:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432956032:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (1135us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099889.601269:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1835
00000100:00100000:1.0:1766099889.601386:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432956032:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099890.175700:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099891.200691:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099892.223688:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099893.247678:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099894.048109:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099894.048174:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099894.048204:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099894.048315:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235712
00000100:00100000:0.0:1766099894.048359:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235584
00000100:00100000:2.0:1766099894.048377:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1689
00000100:00100000:2.0:1766099894.048386:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009375d4cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417235712:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766099894.048394:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1836
00000100:00100000:0.0:1766099894.048397:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417235584:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099894.048471:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417235584:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 73us (297us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099894.048504:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1836
00010000:02020000:2.0:1766099894.048510:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099894.048554:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009375d4cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417235712:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 169us (350us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099894.048569:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1689
00000100:00100000:3.0:1766099894.049059:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235648
00000100:00100000:3.0:1766099894.049103:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2711
00000100:00100000:3.0:1766099894.049108:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417235648:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099894.049323:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ed1b667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417235648:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 213us (1225us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099894.049332:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2711
00010000:00010000:0.0:1766099894.269897:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766099894.718241:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766099894.718251:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766099894.718256:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766099894.718266:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4398
00000100:00080000:3.0:1766099894.718270:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1766099894.718276:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766099894.718302:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000094636470 x1851885432956480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099894.718350:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432956416:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099894.718435:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099894.718505:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099894.718512:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432956544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099894.718545:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099894.718589:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432956480:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099894.718623:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099894.718684:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956416
00000100:00100000:3.0:1766099894.718735:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956544
00000100:00100000:3.0:1766099894.718861:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2712
00000100:00100000:3.0:1766099894.718869:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432956416:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099894.718929:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc6b3d7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432956416:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (496us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099894.718937:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2712
00000100:00100000:3.0:1766099894.718957:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2713
00000100:00100000:3.0:1766099894.718961:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432956544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099894.718986:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbe45651 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432956544:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (442us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099894.718992:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2713
00000100:00100000:3.0:1766099894.719029:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432956544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099894.719038:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432956416:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099894.719104:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432956608:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099894.719137:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099894.719161:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099894.719167:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432956352:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099894.719194:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099894.719222:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956352
00000100:00100000:2.0:1766099894.719249:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1837
00000100:00100000:2.0:1766099894.719253:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432956352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099894.719284:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432956352:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (90us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099894.719290:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1837
00000100:00100000:2.0:1766099894.719322:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432956352:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099894.719345:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956480
00000100:00100000:2.0:1766099894.719362:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956608
00000100:00100000:2.0:1766099894.719381:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1182
00000100:00100000:2.0:1766099894.719385:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432956608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099894.719417:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432956608:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (281us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099894.719423:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1182
00000100:00100000:2.0:1766099894.719433:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1690
00000100:00100000:2.0:1766099894.719436:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432956480:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766099894.719566:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099894.719609:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432956480:12345-0@lo:8:kworker/u8:0.0 Request processed in 171us (985us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099894.719615:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1690
00000100:00080000:2.0:1766099894.719652:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099894.719666:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099894.719671:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432956480:0@lo:8:kworker/u8:0.0
00010000:00080000:2.0:1766099894.719696:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000413b47c6 x1851885432956608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099901 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766099894.719706:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432956608:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766099895.295469:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099896.317713:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099897.343116:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099898.368044:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099899.172815:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099899.172881:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099899.173011:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099899.173145:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235840
00000100:00100000:0.0:1766099899.173217:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2714
00000100:00100000:0.0:1766099899.173224:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c91368f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417235840:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099899.173309:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c91368f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417235840:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 85us (508us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099899.173317:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2714
00000100:00100000:3.0:1766099899.173426:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235776
00000100:00100000:3.0:1766099899.173458:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1838
00000100:00100000:3.0:1766099899.173462:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417235776:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.173517:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417235776:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 54us (637us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099899.173523:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1838
00000100:00100000:1.0:1766099899.173721:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235904
00000100:00100000:1.0:1766099899.173757:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1691
00000100:00100000:1.0:1766099899.173761:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417235904:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099899.173867:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099899.173905:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417235904:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 142us (894us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099899.173912:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1691
00010000:00010000:1.0:1766099899.390113:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099899.846990:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099899.846998:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099899.847004:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099899.847013:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4403
00000100:00080000:2.0:1766099899.847016:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099899.847022:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099899.847024:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766099899.847049:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007e2ba551 x1851885432956800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099899.847063:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432956736:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099899.847131:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432956800:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099899.847173:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099899.847211:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099899.847238:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432956864:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099899.847276:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099899.847278:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956800
00000100:00100000:2.0:1766099899.847333:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1692
00000100:00100000:2.0:1766099899.847339:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432956800:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847391:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432956928:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847429:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:02020000:2.0:1766099899.847435:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099899.847456:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099899.847466:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018d302a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432956672:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099899.847521:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432956800:12345-0@lo:8:kworker/u8:3.0 Request processed in 179us (312us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099899.847562:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:3.0:1766099899.847639:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099899.847643:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099899.847648:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e2ba551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432956800:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847694:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956672
00000100:00100000:3.0:1766099899.847733:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1839
00000100:00100000:3.0:1766099899.847738:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432956672:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847781:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432956672:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (221us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099899.847789:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1839
00000100:00100000:3.0:1766099899.847832:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018d302a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432956672:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847866:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956736
00000100:00100000:3.0:1766099899.847891:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956864
00000100:00100000:3.0:1766099899.847906:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2715
00000100:00100000:3.0:1766099899.847910:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1197d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432956736:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847946:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1197d22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432956736:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (779us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099899.847952:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2715
00000100:00100000:3.0:1766099899.847963:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2716
00000100:00100000:3.0:1766099899.847966:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432956864:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.847988:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3a4d1db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432956864:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (713us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099899.847994:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2716
00000100:00100000:3.0:1766099899.848034:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956928
00000100:00100000:1.0:1766099899.848207:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000413b47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432956864:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099899.848218:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023d2aba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432956736:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099899.851696:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1692
00000100:00100000:3.0:1766099899.852291:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1183
00000100:00100000:3.0:1766099899.852305:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432956928:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099899.854371:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432956928:12345-0@lo:400:kworker/u8:3.0 Request processed in 2057us (6931us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099899.854391:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1183
00010000:00080000:0.0:1766099899.854519:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000094636470 x1851885432956928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099906 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099899.854545:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432956928:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099900.415494:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099901.438450:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099902.461778:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099903.485768:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099904.293744:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099904.293800:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099904.293895:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099904.294017:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417235968
00000100:00100000:2.0:1766099904.294090:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1840
00000100:00100000:2.0:1766099904.294098:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417235968:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099904.294182:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417235968:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 85us (450us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099904.294189:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1840
00000100:00100000:1.0:1766099904.295438:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236032
00000100:00100000:1.0:1766099904.295621:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2717
00000100:00100000:1.0:1766099904.295628:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086a458df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417236032:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099904.295713:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086a458df pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417236032:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 85us (1914us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099904.295722:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2717
00000100:00100000:3.0:1766099904.299746:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236096
00000100:00100000:3.0:1766099904.299814:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1693
00000100:00100000:3.0:1766099904.299821:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417236096:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099904.299931:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099904.300015:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417236096:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 193us (6119us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099904.300023:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1693
00010000:00010000:0.0:1766099904.510073:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099904.963697:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099904.963703:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099904.963706:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099904.963714:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4408
00000100:00080000:1.0:1766099904.963718:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099904.963724:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099904.963745:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000093f0e26e x1851885432957120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099904.963923:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014b7c3cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432957056:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.963984:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099904.964011:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099904.964016:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432957184:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964037:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099904.964059:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432957120:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964082:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099904.964107:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957056
00000100:00100000:1.0:1766099904.964138:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957184
00000100:00100000:1.0:1766099904.964153:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2718
00000100:00100000:1.0:1766099904.964158:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432957056:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964197:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000778ea5b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432957056:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (214us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099904.964203:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2718
00000100:00100000:1.0:1766099904.964219:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2719
00000100:00100000:1.0:1766099904.964222:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432957184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964240:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2b7fd72 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432957184:12345-0@lo:400:kworker/u8:3.0 Request processed in 18us (203us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099904.964245:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2719
00000100:00100000:1.0:1766099904.964263:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957120
00000100:00100000:1.0:1766099904.964278:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1694
00000100:00100000:1.0:1766099904.964281:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e76b9049 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432957120:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099904.964377:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099904.964400:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e76b9049 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432957120:12345-0@lo:8:kworker/u8:3.0 Request processed in 118us (318us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099904.964406:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1694
00000100:00100000:1.0:1766099904.964429:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432956992:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964450:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099904.964463:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099904.964467:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432957248:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964523:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099904.964543:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957248
00000100:00100000:1.0:1766099904.964560:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1184
00000100:00100000:1.0:1766099904.964564:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432957248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964593:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432957248:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (69us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099904.964598:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1184
00000100:00080000:1.0:1766099904.964622:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099904.964625:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099904.964628:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f0e26e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432957120:0@lo:8:kworker/u8:3.0
00010000:00080000:1.0:1766099904.964651:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000070c173f7 x1851885432957248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099911 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099904.964660:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070c173f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432957248:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964677:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432957184:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099904.964685:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014b7c3cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432957056:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099904.964964:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432956992
00000100:00100000:0.0:1766099904.964997:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1841
00000100:00100000:0.0:1766099904.965001:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432956992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099904.965038:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432956992:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (587us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099904.965044:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1841
00000100:00100000:0.0:1766099904.965075:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006baa8a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432956992:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099905.537782:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099906.563620:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099907.583662:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099908.606793:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099909.407494:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099909.407544:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099909.407589:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099909.407687:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236224
00000100:00100000:3.0:1766099909.407795:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236160
00000100:00100000:3.0:1766099909.407826:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1842
00000100:00100000:3.0:1766099909.407833:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417236160:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099909.407896:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417236160:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 63us (351us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099909.407903:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1842
00000100:00100000:1.0:1766099909.408120:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236288
00000100:00100000:1.0:1766099909.408156:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1695
00000100:00100000:1.0:1766099909.408160:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417236288:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099909.408253:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099909.408282:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417236288:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 122us (820us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099909.408289:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1695
00000100:00100000:0.0:1766099909.408501:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2720
00000100:00100000:0.0:1766099909.408507:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417236224:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099909.408560:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023d2aba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417236224:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 52us (970us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099909.408567:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2720
00010000:00010000:0.0:1766099909.632550:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099910.077731:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099910.077738:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099910.077741:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099910.077749:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4413
00000100:00080000:0.0:1766099910.077753:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099910.077758:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099910.077779:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007dcd4c4c x1851885432957440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099910.077871:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432957312:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.077937:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099910.077970:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099910.077975:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432957568:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078000:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099910.078021:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432957376:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078043:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099910.078056:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766099910.078060:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432957504:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078077:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099910.078099:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432957440:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078120:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099910.078147:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957568
00000100:00100000:1.0:1766099910.078179:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1185
00000100:00100000:1.0:1766099910.078184:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432957568:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078222:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432957568:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (221us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099910.078228:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1185
00000100:00100000:1.0:1766099910.078243:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957440
00000100:00100000:1.0:1766099910.078258:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1696
00000100:00100000:1.0:1766099910.078261:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057c568c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432957440:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099910.078355:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099910.078381:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057c568c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432957440:12345-0@lo:8:kworker/u8:0.0 Request processed in 119us (261us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099910.078387:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1696
00000100:00080000:1.0:1766099910.078419:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099910.078422:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099910.078425:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dcd4c4c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432957440:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078451:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957376
00000100:00100000:1.0:1766099910.078519:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957504
00000100:00100000:1.0:1766099910.078537:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2721
00000100:00100000:1.0:1766099910.078540:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000031d74cda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432957376:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078574:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000031d74cda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432957376:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (530us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099910.078579:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2721
00000100:00100000:1.0:1766099910.078588:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2722
00000100:00100000:1.0:1766099910.078591:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060fbe51c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432957504:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078611:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060fbe51c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432957504:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (534us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099910.078615:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2722
00000100:00100000:1.0:1766099910.078641:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000057bb2ef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432957504:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099910.078649:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432957376:0@lo:400:kworker/u8:0.0
00010000:00080000:1.0:1766099910.078670:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000073b447f1 x1851885432957568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099917 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099910.078676:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432957568:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099910.078748:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957312
00000100:00100000:0.0:1766099910.078784:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1843
00000100:00100000:0.0:1766099910.078788:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432957312:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099910.078826:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432957312:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (891us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099910.078832:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1843
00000100:00100000:0.0:1766099910.078863:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432957312:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766099910.654539:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099911.677672:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099912.701839:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099913.725887:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099914.528303:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099914.529349:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099914.529405:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099914.529510:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236480
00000100:00100000:1.0:1766099914.529568:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1697
00000100:00100000:1.0:1766099914.529576:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417236480:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099914.529681:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099914.529751:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417236480:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 176us (347us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099914.529758:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1697
00000100:00100000:3.0:1766099914.531361:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236352
00000100:00100000:3.0:1766099914.531450:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1844
00000100:00100000:3.0:1766099914.531457:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417236352:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099914.531559:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417236352:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 102us (2215us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099914.531568:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1844
00000100:00100000:0.0:1766099914.532557:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236416
00000100:00100000:0.0:1766099914.532629:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2723
00000100:00100000:0.0:1766099914.532635:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000413b47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417236416:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099914.532697:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000413b47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417236416:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 62us (4408us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099914.532704:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2723
00010000:00010000:1.0:1766099914.751033:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099915.198774:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099915.198784:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099915.198788:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099915.198799:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4418
00000100:00080000:2.0:1766099915.198803:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099915.198810:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099915.198841:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004cb5390f x1851885432957760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099915.198937:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432957888:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199044:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099915.199090:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099915.199096:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432957632:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199129:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099915.199161:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432957824:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199191:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099915.199209:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099915.199214:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cb308ba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432957696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199237:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099915.199259:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432957760:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199279:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099915.199317:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957632
00000100:00100000:1.0:1766099915.199366:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1845
00000100:00100000:1.0:1766099915.199373:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432957632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199458:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432957632:12345-0@lo:400:kworker/u8:3.0 Request processed in 85us (329us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099915.199467:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1845
00000100:00100000:1.0:1766099915.199538:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957824
00000100:00100000:1.0:1766099915.199568:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957696
00000100:00100000:1.0:1766099915.199588:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2724
00000100:00100000:1.0:1766099915.199592:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653fe0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432957824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199642:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653fe0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432957824:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (451us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099915.199648:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2724
00000100:00100000:1.0:1766099915.199659:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2725
00000100:00100000:1.0:1766099915.199663:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a2ebda9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432957696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199687:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a2ebda9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432957696:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (449us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099915.199692:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2725
00000100:00100000:1.0:1766099915.199739:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cb308ba9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432957696:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199751:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5105a15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432957824:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199785:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a3c97aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432957632:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199807:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957888
00000100:00100000:1.0:1766099915.199829:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957760
00000100:00100000:1.0:1766099915.199843:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1186
00000100:00100000:1.0:1766099915.199846:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432957888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099915.199878:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432957888:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (835us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099915.199883:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1186
00000100:00100000:1.0:1766099915.199894:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1698
00000100:00100000:1.0:1766099915.199897:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432957760:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099915.199989:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099915.200023:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432957760:12345-0@lo:8:kworker/u8:3.0 Request processed in 124us (743us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099915.200029:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1698
00000100:00080000:1.0:1766099915.200061:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099915.200066:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099915.200069:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cb5390f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432957760:0@lo:8:kworker/u8:3.0
00010000:00080000:1.0:1766099915.200090:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c63c9d57 x1851885432957888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099922 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766099915.200099:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c63c9d57 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432957888:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099915.773973:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099916.799403:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099917.821788:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099918.845863:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099919.648290:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099919.648354:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099919.648409:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099919.649763:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236672
00000100:00100000:2.0:1766099919.649843:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1699
00000100:00100000:2.0:1766099919.649852:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417236672:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766099919.650279:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099919.650382:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417236672:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 530us (1972us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099919.650400:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1699
00000100:00100000:0.0:1766099919.650557:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236608
00000100:00100000:0.0:1766099919.650602:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2726
00000100:00100000:0.0:1766099919.650607:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bec75682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417236608:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099919.650665:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bec75682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417236608:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 54us (2385us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099919.650672:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2726
00000100:00100000:3.0:1766099919.650792:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236544
00000100:00100000:3.0:1766099919.650831:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1846
00000100:00100000:3.0:1766099919.650835:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417236544:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099919.650861:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417236544:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 26us (2508us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099919.650867:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1846
00010000:00010000:0.0:1766099919.875073:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099920.318020:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099920.318028:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099920.318033:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099920.318043:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4423
00000100:00080000:0.0:1766099920.318047:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099920.318055:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099920.318076:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000048d710c3 x1851885432958080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099920.318379:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432957952:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099920.318521:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099920.318561:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766099920.318566:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432958208:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099920.318600:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099920.318642:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432957952
00000100:00100000:2.0:1766099920.318687:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1847
00000100:00100000:2.0:1766099920.318694:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432957952:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099920.318745:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432957952:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (270us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099920.318753:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1847
00000100:00100000:2.0:1766099920.318789:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958208
00000100:00100000:2.0:1766099920.318815:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1187
00000100:00100000:2.0:1766099920.318819:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432958208:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099920.318848:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432958208:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (248us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099920.318854:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1187
00010000:00080000:2.0:1766099920.318890:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009296b030 x1851885432958208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099927 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099920.318904:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432958208:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099920.318913:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432957952:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099920.319522:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062653d94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432958144:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099920.319568:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099920.319592:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099920.319598:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432958016:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099920.319618:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099920.319652:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048d710c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432958080:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099920.319680:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099920.319763:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958144
00000100:00100000:0.0:1766099920.319810:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958016
00000100:00100000:0.0:1766099920.319832:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2727
00000100:00100000:0.0:1766099920.319836:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d784a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432958144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099920.319883:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d784a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432958144:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (315us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099920.319890:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2727
00000100:00100000:0.0:1766099920.319907:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2728
00000100:00100000:0.0:1766099920.319910:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432958016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099920.319933:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000142c5414 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432958016:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (315us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099920.319938:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2728
00000100:00100000:1.0:1766099920.320852:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432958016:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099920.320867:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062653d94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432958144:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099920.321864:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958080
00000100:00100000:2.0:1766099920.321912:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1700
00000100:00100000:2.0:1766099920.322107:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432958080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766099920.322415:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099920.322584:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432958080:12345-0@lo:8:kworker/u8:3.0 Request processed in 479us (2903us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099920.322594:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1700
00000100:00080000:1.0:1766099920.322619:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099920.322624:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099920.322629:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048d710c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432958080:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099920.896448:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099921.918357:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099922.943076:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099923.967242:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099924.766215:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099924.767491:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236864
00000100:00100000:0.0:1766099924.767572:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1701
00000100:00100000:0.0:1766099924.767580:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417236864:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766099924.767706:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099924.767804:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417236864:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 222us (1599us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099924.767812:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1701
00000100:00100000:2.0:1766099924.774122:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099924.774174:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099924.774656:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236800
00000100:00100000:0.0:1766099924.774732:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2729
00000100:00100000:0.0:1766099924.774740:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417236800:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099924.775170:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236736
00000100:00100000:3.0:1766099924.775211:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1848
00000100:00100000:3.0:1766099924.775216:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417236736:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099924.775308:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417236736:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 92us (1134us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099924.775316:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1848
00000100:00100000:0.0:1766099924.775991:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5105a15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417236800:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1247us (1877us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099924.776002:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2729
00010000:00010000:0.0:1766099924.989823:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099925.438709:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099925.438719:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099925.438724:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099925.438734:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4428
00000100:00080000:2.0:1766099925.438738:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099925.438745:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099925.438759:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099925.438772:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000079ab5e9e x1851885432958400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099925.438825:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432958336:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.438905:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099925.438940:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766099925.438944:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432958464:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.438963:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099925.438998:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958336
00000100:00100000:2.0:1766099925.439040:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958464
00000100:00100000:2.0:1766099925.439059:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2730
00000100:00100000:2.0:1766099925.439065:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432958336:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.439112:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df6e7141 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432958336:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (209us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099925.439119:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2730
00000100:00100000:2.0:1766099925.439135:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2731
00000100:00100000:2.0:1766099925.439138:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f78b201d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432958464:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.439154:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f78b201d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432958464:12345-0@lo:400:kworker/u8:0.0 Request processed in 16us (191us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099925.439159:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2731
00000100:00100000:2.0:1766099925.439190:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432958464:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.439201:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432958336:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.439243:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000079ab5e9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432958400:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099925.439244:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432958528:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099925.439270:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099925.439288:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958400
00000100:00100000:1.0:1766099925.439311:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1702
00000100:00100000:1.0:1766099925.439314:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003204340c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432958400:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099925.439418:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099925.439467:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003204340c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432958400:12345-0@lo:8:kworker/u8:0.0 Request processed in 151us (196us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099925.439505:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1702
00000100:00100000:1.0:1766099925.439524:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099925.439548:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000949bc664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432958272:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099925.439575:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099925.439602:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958272
00000100:00100000:1.0:1766099925.439626:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1849
00000100:00100000:1.0:1766099925.439630:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432958272:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099925.439660:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432958272:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (85us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099925.439666:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1849
00000100:00100000:1.0:1766099925.439699:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000949bc664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432958272:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099925.439859:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958528
00000100:00100000:3.0:1766099925.439894:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1188
00000100:00100000:3.0:1766099925.439898:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432958528:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099925.439940:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432958528:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (416us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099925.439946:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1188
00010000:00080000:3.0:1766099925.439970:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000da267b78 x1851885432958528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099932 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099925.439982:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000da267b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432958528:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766099925.442876:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099925.442892:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099925.442902:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000079ab5e9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432958400:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766099926.013954:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099927.037795:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099928.061919:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099929.086614:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099929.886857:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099929.889077:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099929.889196:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236992
00000100:00100000:3.0:1766099929.889283:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2732
00000100:00100000:3.0:1766099929.889290:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000665bb208 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417236992:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099929.889405:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000665bb208 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417236992:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 115us (341us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099929.889414:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2732
00000100:00100000:1.0:1766099929.890742:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417236928
00000100:00100000:0.0:1766099929.891593:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1850
00000100:00100000:0.0:1766099929.891602:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417236928:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099929.891696:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417236928:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 94us (4853us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099929.891704:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1850
00000100:00100000:3.0:1766099929.894025:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099929.894443:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237056
00000100:00100000:1.0:1766099929.894595:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1703
00000100:00100000:1.0:1766099929.894602:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417237056:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766099929.894872:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099929.894964:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417237056:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 362us (950us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099929.894974:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1703
00010000:00010000:1.0:1766099930.111286:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099930.559644:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099930.559652:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099930.559657:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099930.559667:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4433
00000100:00080000:0.0:1766099930.559671:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099930.559679:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099930.559705:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003dd022ac x1851885432958720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099930.559908:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432958848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099930.559997:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099930.560047:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099930.560053:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432958592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099930.560081:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099930.560128:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958592
00000100:00100000:3.0:1766099930.560196:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1851
00000100:00100000:3.0:1766099930.560204:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432958592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099930.560260:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432958592:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (178us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099930.560268:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1851
00000100:00100000:3.0:1766099930.560334:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432958592:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099930.560510:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958848
00000100:00100000:1.0:1766099930.560554:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1189
00000100:00100000:1.0:1766099930.560558:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432958848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099930.560604:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432958848:12345-0@lo:400:kworker/u8:3.0 Request processed in 45us (608us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099930.560622:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1189
00010000:00080000:3.0:1766099930.563288:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b602d77c x1851885432958848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099937 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099930.563325:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432958848:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099930.563509:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432958656:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099930.563585:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432958784:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099930.563586:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099930.563616:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099930.563630:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432958720:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099930.563658:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099930.563707:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958656
00000100:00100000:1.0:1766099930.563769:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958784
00000100:00100000:1.0:1766099930.563790:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2733
00000100:00100000:1.0:1766099930.563796:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065fe2a9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432958656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099930.563853:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065fe2a9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432958656:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (269us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099930.563860:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2733
00000100:00100000:1.0:1766099930.563877:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2734
00000100:00100000:1.0:1766099930.563881:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc100191 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432958784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099930.563910:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc100191 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432958784:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (294us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099930.563916:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2734
00000100:00100000:1.0:1766099930.563940:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432958784:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099930.564020:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432958656:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099930.564961:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958720
00000100:00100000:0.0:1766099930.565005:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1704
00000100:00100000:0.0:1766099930.565010:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432958720:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766099930.565114:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00080000:2.0:1766099930.567251:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099930.567265:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099930.567273:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432958720:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766099930.570437:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432958720:12345-0@lo:8:kworker/u8:3.0 Request processed in 5413us (6765us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099930.570458:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1704
00010000:00010000:0.0:1766099931.133893:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099932.157754:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099933.182095:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099934.205892:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099935.006160:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099935.006216:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099935.006252:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099935.006399:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237184
00000100:00100000:1.0:1766099935.006418:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237120
00000100:00100000:1.0:1766099935.006472:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1852
00000100:00100000:1.0:1766099935.006516:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417237120:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099935.006521:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2735
00000100:00100000:3.0:1766099935.006532:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417237184:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099935.006616:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417237120:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 101us (399us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099935.006626:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1852
00000100:00100000:0.0:1766099935.007019:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237248
00000100:00100000:0.0:1766099935.007056:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1705
00000100:00100000:0.0:1766099935.007061:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417237248:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099935.007166:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099935.007316:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da267b78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417237184:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 785us (1166us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099935.007323:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2735
00000100:00100000:0.0:1766099935.007405:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417237248:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 342us (1153us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099935.007412:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1705
00010000:00010000:0.0:1766099935.229731:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099935.680947:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099935.680956:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099935.680960:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099935.680970:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4439
00000100:00080000:2.0:1766099935.680976:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099935.680984:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099935.681018:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004ac1f2ac x1851885432959040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099935.681072:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432959040:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766099935.681157:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099935.681215:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959040
00000100:00100000:2.0:1766099935.681257:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1706
00000100:00100000:2.0:1766099935.681265:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000766b1095 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432959040:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766099935.681376:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766099935.681423:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000766b1095 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432959040:12345-0@lo:8:kworker/u8:0.0 Request processed in 158us (268us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099935.681430:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1706
00000100:00080000:2.0:1766099935.681507:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099935.681512:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099935.681516:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432959040:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681602:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b47533c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432959104:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681642:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099935.681675:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099935.681681:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432958976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681703:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099935.681733:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959104
00000100:00100000:1.0:1766099935.681768:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958976
00000100:00100000:1.0:1766099935.681789:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2736
00000100:00100000:1.0:1766099935.681793:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432959104:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681837:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb308ba9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432959104:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (194us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099935.681843:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2736
00000100:00100000:1.0:1766099935.681859:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2737
00000100:00100000:1.0:1766099935.681862:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432958976:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681888:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006baa8a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432958976:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (184us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099935.681894:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2737
00000100:00100000:1.0:1766099935.681917:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432958912:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681949:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099935.681966:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099935.681970:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9a5f5e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432959168:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.681993:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099935.682021:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432958976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099935.682031:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b47533c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432959104:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099935.682514:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432958912
00000100:00100000:2.0:1766099935.682552:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1853
00000100:00100000:2.0:1766099935.682556:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432958912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099935.682593:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432958912:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (644us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099935.682599:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1853
00000100:00100000:2.0:1766099935.682627:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959168
00000100:00100000:2.0:1766099935.682655:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1190
00000100:00100000:2.0:1766099935.682658:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432959168:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099935.682683:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432959168:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (689us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099935.682688:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1190
00010000:00080000:0.0:1766099935.683050:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d9a5f5e2 x1851885432959168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099942 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099935.683063:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9a5f5e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432959168:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099935.683073:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432958912:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099936.253759:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099937.279601:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099938.303242:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099939.325910:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099940.128286:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099940.128351:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099940.128390:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099940.128539:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237440
00000100:00100000:3.0:1766099940.128589:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1707
00000100:00100000:3.0:1766099940.128598:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417237440:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099940.128705:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099940.128782:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417237440:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 184us (508us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099940.128791:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1707
00000100:00100000:3.0:1766099940.128820:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237376
00000100:00100000:3.0:1766099940.128857:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2738
00000100:00100000:3.0:1766099940.128861:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b219b5c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417237376:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099940.128908:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b219b5c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417237376:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (518us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099940.128914:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2738
00000100:00100000:1.0:1766099940.129203:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237312
00000100:00100000:1.0:1766099940.129253:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1854
00000100:00100000:1.0:1766099940.129258:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417237312:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099940.129289:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417237312:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 30us (939us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099940.129296:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1854
00010000:00010000:0.0:1766099940.351677:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099940.798684:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099940.798692:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099940.798696:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099940.798705:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4444
00000100:00080000:0.0:1766099940.798709:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099940.798715:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099940.798737:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f5400e13 x1851885432959360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099940.798795:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432959296:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.798876:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099940.798915:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766099940.798921:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432959424:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.798944:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099940.798973:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432959360:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099940.799001:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099940.799042:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959296
00000100:00100000:0.0:1766099940.799087:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959424
00000100:00100000:0.0:1766099940.799106:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2739
00000100:00100000:0.0:1766099940.799112:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d710c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432959296:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.799161:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d710c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432959296:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (287us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099940.799169:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2739
00000100:00100000:0.0:1766099940.799193:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2740
00000100:00100000:0.0:1766099940.799196:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432959424:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.799219:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062653d94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432959424:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (275us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099940.799224:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2740
00000100:00100000:0.0:1766099940.799257:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959360
00000100:00100000:0.0:1766099940.799276:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1708
00000100:00100000:0.0:1766099940.799280:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432959360:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766099940.799384:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099940.799418:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432959360:12345-0@lo:8:kworker/u8:0.0 Request processed in 137us (417us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099940.799425:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1708
00000100:00080000:0.0:1766099940.799461:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099940.799465:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099940.799470:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432959360:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099940.799527:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432959424:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.799537:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432959296:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099940.799765:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432959232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099940.799807:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099940.799843:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766099940.799849:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432959488:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099940.799874:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099940.799905:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959488
00000100:00100000:3.0:1766099940.799932:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1191
00000100:00100000:3.0:1766099940.799936:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432959488:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099940.799969:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432959488:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (94us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099940.799976:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1191
00010000:00080000:3.0:1766099940.800000:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005bf7662d x1851885432959488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099947 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099940.800010:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432959488:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.800217:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959232
00000100:00100000:0.0:1766099940.800258:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1855
00000100:00100000:0.0:1766099940.800262:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432959232:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099940.800752:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432959232:12345-0@lo:400:kworker/u8:0.0 Request processed in 487us (944us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099940.800760:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1855
00000100:00100000:3.0:1766099940.804142:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432959232:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766099941.374005:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099942.397771:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099943.422700:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099944.448061:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099945.244734:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099945.244778:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099945.244854:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099945.244953:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237632
00000100:00100000:3.0:1766099945.245005:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1709
00000100:00100000:3.0:1766099945.245012:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417237632:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099945.245125:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099945.245210:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417237632:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 187us (346us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099945.245217:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1709
00000100:00100000:0.0:1766099945.245283:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237504
00000100:00100000:0.0:1766099945.245331:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1856
00000100:00100000:0.0:1766099945.245336:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417237504:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099945.245389:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417237504:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 52us (665us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099945.245397:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1856
00000100:00100000:1.0:1766099945.245834:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237568
00000100:00100000:1.0:1766099945.245880:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2741
00000100:00100000:1.0:1766099945.245884:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007331255b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417237568:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099945.245936:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007331255b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417237568:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 51us (1158us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099945.245943:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2741
00010000:00010000:1.0:1766099945.469826:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099945.919059:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099945.919178:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432959616:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099945.919279:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766099945.919339:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099945.919345:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099945.919350:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099945.919355:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4449
00000100:00080000:0.0:1766099945.919358:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099945.919365:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099945.919392:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f5400e13 x1851885432959680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099945.919928:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432959552:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099945.919978:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099945.920031:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959616
00000100:00100000:2.0:1766099945.920084:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2742
00000100:00100000:2.0:1766099945.920092:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de379e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432959616:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099945.920243:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959552
00000100:00100000:3.0:1766099945.920279:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1857
00000100:00100000:3.0:1766099945.920283:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432959552:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099945.920326:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432959552:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (348us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099945.920334:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1857
00000100:00100000:3.0:1766099945.920390:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432959552:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099945.921235:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432959680:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099945.921289:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099945.921332:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432959744:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099945.921360:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099945.921394:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959680
00000100:00100000:1.0:1766099945.921425:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1710
00000100:00100000:1.0:1766099945.921431:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432959680:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099945.921562:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099945.921601:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432959680:12345-0@lo:8:kworker/u8:0.0 Request processed in 169us (313us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099945.921608:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1710
00000100:00080000:1.0:1766099945.921646:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099945.921650:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099945.921654:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5400e13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432959680:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099945.921677:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959744
00000100:00100000:1.0:1766099945.921704:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2743
00000100:00100000:1.0:1766099945.921707:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432959744:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099945.921747:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070c173f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432959744:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (386us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099945.921753:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2743
00000100:00100000:1.0:1766099945.921782:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432959744:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099945.922042:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de379e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432959616:12345-0@lo:400:kworker/u8:0.0 Request processed in 1948us (2762us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099945.922050:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2742
00000100:00100000:2.0:1766099945.922223:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432959808:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099945.922256:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099945.922337:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432959616:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099945.922436:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959808
00000100:00100000:0.0:1766099945.922474:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1192
00000100:00100000:0.0:1766099945.922495:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432959808:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099945.922546:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432959808:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (290us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099945.922553:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1192
00010000:00080000:3.0:1766099945.922621:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009296b030 x1851885432959808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099952 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766099945.922633:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432959808:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766099946.493979:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099947.519461:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099948.541855:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099949.566565:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099950.364704:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099950.364758:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099950.364807:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099950.365043:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237760
00000100:00100000:3.0:1766099950.365119:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2744
00000100:00100000:3.0:1766099950.365128:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa6f101e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417237760:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099950.365237:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa6f101e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417237760:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 108us (543us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099950.365245:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2744
00000100:00100000:1.0:1766099950.365740:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237824
00000100:00100000:0.0:1766099950.366414:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1711
00000100:00100000:0.0:1766099950.366422:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417237824:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099950.366591:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099950.366660:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417237824:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 238us (1902us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099950.366669:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1711
00000100:00100000:0.0:1766099950.366747:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237696
00000100:00100000:0.0:1766099950.366793:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1858
00000100:00100000:0.0:1766099950.366798:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417237696:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099950.366833:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417237696:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 35us (2027us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099950.366842:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1858
00010000:00010000:0.0:1766099950.592548:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099951.040053:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099951.040063:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099951.040068:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099951.040081:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4454
00000100:00080000:2.0:1766099951.040086:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099951.040094:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099951.040123:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b0591111 x1851885432960000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099951.040414:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432960064:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099951.040432:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432959936:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099951.040517:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9a5f5e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432960128:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099951.040575:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055b9c8f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432959872:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099951.041592:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099951.042596:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099951.042621:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099951.042688:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960000:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099951.042701:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959936
00000100:00100000:0.0:1766099951.042740:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099951.042767:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2745
00000100:00100000:2.0:1766099951.042776:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000772d840f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432959936:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099951.042780:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960128
00000100:00100000:0.0:1766099951.042811:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960000
00000100:00100000:0.0:1766099951.042832:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1193
00000100:00100000:0.0:1766099951.042837:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432960128:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099951.042855:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000772d840f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432959936:12345-0@lo:400:kworker/u8:3.0 Request processed in 79us (1268us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099951.042863:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2745
00000100:00100000:0.0:1766099951.042888:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432960128:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (297us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099951.042895:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1193
00000100:00100000:1.0:1766099951.043252:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099951.043589:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432959872
00000100:00100000:0.0:1766099951.045740:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1712
00000100:00100000:0.0:1766099951.045755:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432960000:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099951.045772:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1859
00000100:00100000:3.0:1766099951.045779:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432959872:12345-0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766099951.045863:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099951.046019:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432959872:12345-0@lo:400:kworker/u8:3.0 Request processed in 239us (3400us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099951.046027:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1859
00000100:00100000:3.0:1766099951.046149:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432959936:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099951.046221:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055b9c8f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432959872:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766099951.046274:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d9a5f5e2 x1851885432960128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099958 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099951.046286:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9a5f5e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432960128:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099951.046559:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432960000:12345-0@lo:8:kworker/u8:3.0 Request processed in 803us (3818us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099951.046568:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1712
00000100:00080000:3.0:1766099951.047011:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099951.047016:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099951.047020:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0591111 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960000:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099951.048772:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960064
00000100:00100000:2.0:1766099951.049302:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2746
00000100:00100000:2.0:1766099951.049310:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432960064:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099951.049438:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f0e26e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432960064:12345-0@lo:400:kworker/u8:3.0 Request processed in 128us (6186us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099951.049447:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2746
00000100:00100000:3.0:1766099951.049515:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000580eab13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432960064:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766099951.614555:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099952.641549:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099953.663262:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099954.686087:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099955.490577:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099955.492020:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099955.492381:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237888
00000100:00100000:0.0:1766099955.492592:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1860
00000100:00100000:0.0:1766099955.492600:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417237888:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766099955.492702:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417237888:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 102us (2237us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099955.492710:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1860
00000100:00100000:2.0:1766099955.493357:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417237952
00000100:00100000:3.0:1766099955.493400:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099955.493409:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2747
00000100:00100000:2.0:1766099955.493416:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084b59448 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417237952:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099955.493455:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238016
00000100:00100000:2.0:1766099955.493470:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084b59448 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417237952:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 54us (1457us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099955.493510:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1713
00000100:00100000:3.0:1766099955.493515:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417238016:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099955.493621:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099955.493657:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417238016:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 141us (260us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099955.493664:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1713
00000100:00100000:2.0:1766099955.494001:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2747
00010000:00010000:0.0:1766099955.710675:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099956.158114:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:1.0:1766099956.158706:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099956.158714:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099956.158718:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099956.158728:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4459
00000100:00080000:1.0:1766099956.158732:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099956.158739:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099956.158767:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ebfd82a3 x1851885432960320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099956.158884:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432960256:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099956.158966:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099956.158997:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099956.159001:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000080a7d48c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432960384:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099956.159018:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099956.159073:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960256
00000100:00100000:3.0:1766099956.159115:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960384
00000100:00100000:3.0:1766099956.159149:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2748
00000100:00100000:3.0:1766099956.159156:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6a67406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432960256:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099956.159211:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6a67406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432960256:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (245us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099956.159215:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ebfd82a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960320:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099956.159222:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2748
00000100:00100000:3.0:1766099956.159243:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2749
00000100:00100000:3.0:1766099956.159247:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006de96a83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432960384:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099956.159265:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099956.159282:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006de96a83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432960384:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (262us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099956.159289:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2749
00000100:00100000:3.0:1766099956.159337:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000080a7d48c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432960384:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099956.159364:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061093366 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432960256:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099956.159395:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960320
00000100:00100000:2.0:1766099956.159428:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1714
00000100:00100000:2.0:1766099956.159433:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acbe6453 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432960320:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766099956.159553:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099956.159586:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ad04e0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432960448:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099956.159605:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acbe6453 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432960320:12345-0@lo:8:kworker/u8:0.0 Request processed in 171us (342us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766099956.159613:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1714
00000100:00100000:0.0:1766099956.159626:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099956.159649:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432960192:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099956.159681:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099956.159717:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960192
00000100:00100000:0.0:1766099956.159746:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1861
00000100:00100000:0.0:1766099956.159750:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432960192:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099956.159782:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960448
00000100:00100000:0.0:1766099956.159785:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432960192:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (105us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099956.159792:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1861
00000100:00100000:2.0:1766099956.159809:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1194
00000100:00100000:2.0:1766099956.159815:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432960448:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099956.159840:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000037d49a48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432960192:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099956.159848:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432960448:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (223us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099956.159855:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1194
00010000:00080000:0.0:1766099956.159860:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002ad04e0d x1851885432960448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099963 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099956.159869:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ad04e0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432960448:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766099956.164316:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099956.164334:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099956.164345:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ebfd82a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960320:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766099956.733721:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099957.757829:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099958.784814:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099959.805850:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099960.605459:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099960.609428:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238208
00000100:00100000:0.0:1766099960.609548:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1715
00000100:00100000:0.0:1766099960.609558:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2734238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417238208:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766099960.609688:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099960.609788:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2734238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417238208:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 231us (4341us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099960.609796:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1715
00000100:00100000:1.0:1766099960.610195:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099960.615303:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099960.615434:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238080
00000100:00100000:0.0:1766099960.615578:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238144
00000100:00100000:0.0:1766099960.615644:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2750
00000100:00100000:0.0:1766099960.615651:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000483c3824 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238144:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099960.615763:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000483c3824 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238144:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 111us (5570us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099960.615771:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2750
00000100:00100000:3.0:1766099960.617544:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1862
00000100:00100000:3.0:1766099960.617560:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238080:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099960.617664:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238080:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 106us (2372us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099960.617673:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1862
00010000:00010000:1.0:1766099960.829740:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099961.278057:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099961.278066:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099961.278070:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099961.278081:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4464
00000100:00080000:2.0:1766099961.278094:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099961.278102:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099961.278130:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007d494536 x1851885432960640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099961.278361:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432960768:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099961.278454:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099961.278529:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766099961.278536:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000766b1095 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432960512:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099961.278576:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099961.278695:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d494536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960640:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099961.278728:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099961.278770:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960768
00000100:00100000:0.0:1766099961.278815:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960640
00000100:00100000:0.0:1766099961.278832:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1195
00000100:00100000:0.0:1766099961.278839:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432960768:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099961.278889:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432960768:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (437us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099961.278896:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1195
00000100:00100000:0.0:1766099961.278918:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1716
00000100:00100000:0.0:1766099961.278922:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432960640:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766099961.279132:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099961.279172:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432960640:12345-0@lo:8:kworker/u8:0.0 Request processed in 248us (443us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099961.279179:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1716
00000100:00080000:0.0:1766099961.279216:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766099961.279220:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766099961.279225:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d494536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960640:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099961.280684:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432960576:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099961.280765:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099961.280796:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099961.280803:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086b72066 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432960704:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099961.280824:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099961.280876:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960512
00000100:00100000:3.0:1766099961.280926:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1863
00000100:00100000:3.0:1766099961.280933:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432960512:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099961.281151:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432960512:12345-0@lo:400:kworker/u8:0.0 Request processed in 217us (2574us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099961.281160:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1863
00000100:00100000:1.0:1766099961.281550:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960576
00000100:00100000:1.0:1766099961.281601:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960704
00000100:00100000:1.0:1766099961.281622:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2751
00000100:00100000:1.0:1766099961.281627:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096d55fda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432960576:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099961.281677:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096d55fda pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432960576:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (913us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099961.281683:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2751
00000100:00100000:1.0:1766099961.281696:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2752
00000100:00100000:1.0:1766099961.281699:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432960704:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099961.281725:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000492edb9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432960704:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (901us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099961.281732:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2752
00000100:00100000:2.0:1766099961.282551:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000766b1095 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432960512:0@lo:400:kworker/u8:0.0
00010000:00080000:2.0:1766099961.282573:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c2dfefc2 x1851885432960768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099968 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766099961.282587:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432960768:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099961.283819:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086b72066 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432960704:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099961.283841:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432960576:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766099961.855275:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099962.878080:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099963.901684:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099964.925704:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099965.727397:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099965.728336:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238272
00000100:00100000:1.0:1766099965.728352:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099965.728380:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1864
00000100:00100000:2.0:1766099965.728387:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238272:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099965.728411:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099965.728438:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238336
00000100:00100000:2.0:1766099965.728456:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238272:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 68us (1066us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099965.728463:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1864
00000100:00100000:1.0:1766099965.728530:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2753
00000100:00100000:1.0:1766099965.728535:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee6355f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238336:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099965.728566:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee6355f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238336:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 30us (155us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099965.728572:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2753
00000100:00100000:3.0:1766099965.728659:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238400
00000100:00100000:3.0:1766099965.728690:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1717
00000100:00100000:3.0:1766099965.728695:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005001f4c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238400:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766099965.728786:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099965.728829:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005001f4c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238400:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 133us (477us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099965.728836:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1717
00010000:00010000:0.0:1766099965.949767:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766099966.397750:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099966.397756:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099966.397759:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099966.397767:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4469
00000100:00080000:2.0:1766099966.397770:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099966.397775:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099966.397792:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009b47533c x1851885432960960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099966.397834:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086b72066 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432961088:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099966.397897:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099966.397926:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b47533c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960960:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766099966.397927:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099966.397932:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432960832:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099966.397986:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000766b1095 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432960896:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099966.398039:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099966.398081:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432961024:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099966.398106:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099966.398140:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961024
00000100:00100000:0.0:1766099966.398185:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961088
00000100:00100000:1.0:1766099966.398186:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2754
00000100:00100000:1.0:1766099966.398192:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034f090f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432961024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099966.398205:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960960
00000100:00100000:0.0:1766099966.398219:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1196
00000100:00100000:0.0:1766099966.398222:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432961088:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099966.398235:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034f090f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432961024:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (129us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099966.398242:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2754
00000100:00100000:0.0:1766099966.398251:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432961088:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (356us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099966.398256:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1196
00000100:00100000:0.0:1766099966.398268:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1718
00000100:00100000:0.0:1766099966.398271:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432960960:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099966.398276:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432961024:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766099966.398368:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099966.398406:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432960960:12345-0@lo:8:kworker/u8:3.0 Request processed in 134us (367us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099966.398412:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1718
00000100:00100000:2.0:1766099966.398595:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:2.0:1766099966.398626:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000086b72066 x1851885432961088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099973 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766099966.398634:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086b72066 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432961088:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099966.398713:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960832
00000100:00100000:0.0:1766099966.398780:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1865
00000100:00100000:0.0:1766099966.398784:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432960832:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099966.398816:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432960832:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (221us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099966.398822:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1865
00000100:00100000:3.0:1766099966.398868:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099966.398918:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432960832:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099966.398996:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432960896
00000100:00100000:3.0:1766099966.399686:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2755
00000100:00100000:3.0:1766099966.399690:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432960896:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099966.399744:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3ef1f4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432960896:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (876us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099966.399750:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2755
00000100:00100000:2.0:1766099966.399827:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000766b1095 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432960896:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766099966.400014:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099966.400018:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099966.400021:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b47533c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432960960:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766099966.974359:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099967.997715:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099969.022293:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099970.047965:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099970.847579:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099970.847644:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099970.847687:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238592
00000100:00100000:1.0:1766099970.847754:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099970.847754:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1719
00000100:00100000:3.0:1766099970.847766:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238592:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:0.0:1766099970.847861:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238528
00010000:02020000:3.0:1766099970.847874:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099970.847913:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2756
00000100:00100000:0.0:1766099970.847919:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002569b802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417238528:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766099970.847956:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238592:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 190us (387us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099970.847966:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1719
00000100:00100000:0.0:1766099970.847971:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002569b802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417238528:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 51us (222us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099970.847979:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2756
00000100:00100000:2.0:1766099970.848152:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238464
00000100:00100000:2.0:1766099970.848194:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1866
00000100:00100000:2.0:1766099970.848198:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238464:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099970.848244:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238464:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (601us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099970.848251:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1866
00010000:00010000:0.0:1766099971.075395:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099971.517923:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099971.517931:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099971.517935:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099971.517942:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4474
00000100:00080000:1.0:1766099971.517944:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099971.517950:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099971.517969:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000007f24045 x1851885432961280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766099971.518036:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ebfd82a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432961344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518095:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099971.518123:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099971.518129:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432961216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518170:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099971.518192:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007f24045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432961280:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518216:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099971.518246:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961344
00000100:00100000:1.0:1766099971.518282:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961216
00000100:00100000:1.0:1766099971.518298:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2757
00000100:00100000:1.0:1766099971.518303:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a7d48c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432961344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518343:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a7d48c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432961344:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (249us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099971.518350:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2757
00000100:00100000:1.0:1766099971.518363:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2758
00000100:00100000:1.0:1766099971.518366:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ebb20fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432961216:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518387:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ebb20fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432961216:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (218us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099971.518393:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2758
00000100:00100000:1.0:1766099971.518410:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961280
00000100:00100000:1.0:1766099971.518428:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1720
00000100:00100000:1.0:1766099971.518432:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3e125c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432961280:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099971.518555:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099971.518587:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3e125c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432961280:12345-0@lo:8:kworker/u8:3.0 Request processed in 154us (371us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099971.518594:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1720
00000100:00080000:1.0:1766099971.518634:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099971.518638:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099971.518641:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007f24045 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432961280:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518669:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c97d50d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432961216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.518677:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ebfd82a3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432961344:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099971.518750:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432961152:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099971.518785:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099971.518819:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766099971.518824:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ad04e0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432961408:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099971.518855:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099971.520123:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961408
00000100:00100000:1.0:1766099971.520186:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1197
00000100:00100000:1.0:1766099971.520190:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432961408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.520253:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432961408:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (1409us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099971.520259:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1197
00000100:00100000:1.0:1766099971.520281:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961152
00000100:00100000:1.0:1766099971.520310:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1867
00000100:00100000:1.0:1766099971.520313:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432961152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099971.520337:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432961152:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (1552us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099971.520341:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1867
00010000:00080000:0.0:1766099971.523092:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002ad04e0d x1851885432961408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099978 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766099971.523112:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ad04e0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432961408:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099971.523123:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432961152:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099972.095869:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099973.118399:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099974.141699:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099975.166038:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099975.965780:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099975.966739:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099975.966858:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238656
00000100:00100000:0.0:1766099975.967515:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1868
00000100:00100000:0.0:1766099975.967525:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238656:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099975.967644:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238656:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 118us (1879us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099975.967654:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1868
00000100:00100000:1.0:1766099975.967836:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238720
00000100:00100000:1.0:1766099975.967897:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2759
00000100:00100000:1.0:1766099975.967901:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000766b1095 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238720:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099975.967941:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000766b1095 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238720:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 38us (1207us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099975.967948:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2759
00000100:00100000:3.0:1766099975.973863:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099975.974635:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238784
00000100:00100000:1.0:1766099975.974710:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1721
00000100:00100000:1.0:1766099975.974719:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238784:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099975.974834:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099975.975583:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238784:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 859us (1732us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099975.975599:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1721
00010000:00010000:1.0:1766099976.191055:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099976.637751:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099976.637759:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099976.637763:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099976.637770:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4479
00000100:00080000:1.0:1766099976.637775:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099976.637781:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099976.637806:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000632c907e x1851885432961600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099976.637887:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432961728:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099976.637916:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432961664:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099976.638069:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000053dd58eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432961472:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099976.638549:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099976.638582:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766099976.638589:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006131d46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432961536:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099976.638615:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099976.638650:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432961600:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099976.638681:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099976.638835:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961664
00000100:00100000:1.0:1766099976.638877:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961536
00000100:00100000:1.0:1766099976.638896:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2760
00000100:00100000:1.0:1766099976.638902:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ad04e0d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432961664:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099976.638950:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ad04e0d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432961664:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (403us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099976.638957:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2760
00000100:00100000:1.0:1766099976.638984:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2761
00000100:00100000:1.0:1766099976.638987:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebfd82a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432961536:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099976.639014:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebfd82a3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432961536:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (398us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099976.639019:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2761
00000100:00100000:1.0:1766099976.639040:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961600
00000100:00100000:1.0:1766099976.639058:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1722
00000100:00100000:1.0:1766099976.639061:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432961600:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766099976.639153:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099976.639185:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432961600:12345-0@lo:8:kworker/u8:3.0 Request processed in 123us (505us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099976.639191:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1722
00000100:00080000:1.0:1766099976.639225:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099976.639228:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099976.639232:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432961600:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766099976.639379:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006131d46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432961536:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099976.639389:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432961664:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766099976.643215:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099976.643989:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961728
00000100:00100000:0.0:1766099976.644040:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099976.644089:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1198
00000100:00100000:3.0:1766099976.644099:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432961728:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099976.644209:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432961728:12345-0@lo:400:kworker/u8:3.0 Request processed in 111us (1004us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099976.644218:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1198
00000100:00100000:3.0:1766099976.644259:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961472
00000100:00100000:3.0:1766099976.644297:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1869
00010000:00080000:0.0:1766099976.644301:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000065bd5068 x1851885432961728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099983 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099976.644308:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432961472:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766099976.644343:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432961728:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099976.644382:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432961472:12345-0@lo:400:kworker/u8:3.0 Request processed in 76us (350us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099976.644392:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1869
00000100:00100000:3.0:1766099976.644456:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000053dd58eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432961472:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099977.221220:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099978.237715:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099979.261716:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099980.285707:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099981.089694:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099981.089753:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099981.089788:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099981.089907:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238976
00000100:00100000:1.0:1766099981.089966:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1723
00000100:00100000:1.0:1766099981.089973:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238976:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099981.090119:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099981.090190:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417238976:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 216us (506us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099981.090206:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1723
00000100:00100000:0.0:1766099981.090578:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238848
00000100:00100000:0.0:1766099981.090623:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1870
00000100:00100000:0.0:1766099981.090628:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238848:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099981.090682:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417238848:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 53us (929us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099981.090689:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1870
00000100:00100000:3.0:1766099981.091820:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417238912
00000100:00100000:3.0:1766099981.091888:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2762
00000100:00100000:3.0:1766099981.091895:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238912:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766099981.091975:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000580eab13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417238912:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (2185us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099981.091983:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2762
00010000:00010000:0.0:1766099981.311374:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766099981.757978:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099981.757987:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099981.757992:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099981.758001:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4485
00000100:00080000:0.0:1766099981.758005:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099981.758011:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099981.758032:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005a16d6f1 x1851885432961920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099981.758124:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432962048:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099981.758202:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099981.758243:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766099981.758248:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432961792:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099981.758274:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099981.758379:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432961856:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099981.758410:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099981.758429:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099981.758433:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432961984:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099981.758449:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099981.758507:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961856
00000100:00100000:3.0:1766099981.758563:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961984
00000100:00100000:3.0:1766099981.758581:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2763
00000100:00100000:3.0:1766099981.758587:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432961856:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099981.758633:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de717044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432961856:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (223us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099981.758640:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2763
00000100:00100000:3.0:1766099981.758654:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2764
00000100:00100000:3.0:1766099981.758657:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432961984:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099981.758677:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a912b62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432961984:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (229us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099981.758683:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2764
00000100:00100000:3.0:1766099981.758720:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009296b030 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432961984:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099981.758728:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432961856:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099981.758856:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962048
00000100:00100000:0.0:1766099981.758891:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1199
00000100:00100000:0.0:1766099981.758895:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432962048:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099981.758929:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432962048:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (729us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099981.758944:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1199
00010000:00080000:0.0:1766099981.758973:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b602d77c x1851885432962048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099988 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099981.758982:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432962048:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099981.761123:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432961920:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099981.761207:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099981.761275:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961792
00000100:00100000:1.0:1766099981.761319:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1871
00000100:00100000:1.0:1766099981.761324:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432961792:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099981.761374:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432961792:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (3099us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099981.761380:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1871
00000100:00100000:1.0:1766099981.761432:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432961920
00000100:00100000:1.0:1766099981.761447:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1724
00000100:00100000:1.0:1766099981.761450:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432961920:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766099981.761602:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099981.761635:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432961920:12345-0@lo:8:kworker/u8:0.0 Request processed in 184us (429us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099981.761641:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1724
00000100:00080000:1.0:1766099981.761674:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766099981.761678:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766099981.761682:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432961920:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099981.761708:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432961792:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766099982.334396:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099983.358016:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099984.381866:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099985.405876:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766099986.206348:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766099986.206403:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099986.206530:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239040
00000100:00100000:1.0:1766099986.206552:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239104
00000100:00100000:0.0:1766099986.206608:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1872
00000100:00100000:0.0:1766099986.206617:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239040:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099986.206636:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2765
00000100:00100000:1.0:1766099986.206644:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9a5f5e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417239104:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766099986.206696:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239040:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (353us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099986.206706:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1872
00000100:00100000:1.0:1766099986.206728:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9a5f5e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417239104:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 82us (324us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099986.206742:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2765
00000100:00100000:0.0:1766099986.206828:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099986.208271:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239168
00000100:00100000:1.0:1766099986.208336:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1725
00000100:00100000:1.0:1766099986.208343:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b602d77c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417239168:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099986.208469:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099986.208588:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b602d77c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417239168:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 244us (1760us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099986.208596:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1725
00010000:00010000:0.0:1766099986.432175:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766099986.880373:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055b9c8f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432962112:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099986.880456:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099986.880575:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962112
00000100:00100000:3.0:1766099986.880622:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1873
00000100:00100000:3.0:1766099986.880627:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432962112:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766099986.881237:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766099986.881243:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766099986.881246:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766099986.881250:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4490
00000100:00080000:2.0:1766099986.881253:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:2.0:1766099986.881257:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766099986.881277:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009b47533c x1851885432962240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766099986.881439:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432962112:12345-0@lo:400:kworker/u8:3.0 Request processed in 810us (984us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099986.881448:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1873
00000100:00100000:3.0:1766099986.881513:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432962176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881547:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099986.881566:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099986.881571:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432962304:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881591:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099986.881620:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962176
00000100:00100000:3.0:1766099986.881647:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962304
00000100:00100000:3.0:1766099986.881662:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2766
00000100:00100000:3.0:1766099986.881666:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432962176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881704:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084be1e46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432962176:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (156us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099986.881711:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2766
00000100:00100000:3.0:1766099986.881723:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2767
00000100:00100000:3.0:1766099986.881727:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037d49a48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432962304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881755:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037d49a48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432962304:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (164us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099986.881761:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2767
00000100:00100000:3.0:1766099986.881805:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ac1f2ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432962304:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881815:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c2131aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432962176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881938:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b47533c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432962240:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099986.881965:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099986.882002:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962240
00000100:00100000:3.0:1766099986.882022:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1726
00000100:00100000:3.0:1766099986.882026:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432962240:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766099986.882125:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099986.882157:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432962240:12345-0@lo:8:kworker/u8:3.0 Request processed in 130us (191us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099986.882162:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1726
00000100:00080000:3.0:1766099986.882182:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099986.882185:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099986.882188:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b47533c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432962240:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766099986.882313:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432962368:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.882334:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099986.882354:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962368
00000100:00100000:3.0:1766099986.882374:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1200
00000100:00100000:3.0:1766099986.882377:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432962368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766099986.882403:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432962368:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (69us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099986.882408:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1200
00010000:00080000:3.0:1766099986.882424:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000020b6667c x1851885432962368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099993 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766099986.882432:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432962368:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766099986.884306:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055b9c8f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432962112:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766099987.454580:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099988.480525:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099989.501755:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099990.527039:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766099991.325597:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099991.325634:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766099991.325676:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099991.326054:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239360
00000100:00100000:3.0:1766099991.326098:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1727
00000100:00100000:3.0:1766099991.326104:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417239360:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766099991.326201:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099991.326262:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417239360:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 156us (627us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099991.326270:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1727
00000100:00100000:1.0:1766099991.326453:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239232
00000100:00100000:1.0:1766099991.326503:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1874
00000100:00100000:1.0:1766099991.326568:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239232:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766099991.326607:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239232:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 99us (1019us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099991.326613:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1874
00000100:00100000:2.0:1766099991.329167:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239296
00000100:00100000:2.0:1766099991.329234:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2768
00000100:00100000:2.0:1766099991.329241:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417239296:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766099991.329316:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5400e13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417239296:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 74us (3640us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099991.329323:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2768
00010000:00010000:1.0:1766099991.549786:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766099991.999337:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766099991.999343:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766099991.999345:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766099991.999353:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4495
00000100:00080000:1.0:1766099991.999356:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766099991.999361:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766099991.999379:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c8b37342 x1851885432962560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766099991.999469:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432962688:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099991.999542:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099991.999576:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766099991.999581:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432962432:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099991.999615:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099992.000393:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962688
00000100:00100000:1.0:1766099992.000444:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1201
00000100:00100000:1.0:1766099992.000450:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432962688:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099992.000520:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432962688:12345-0@lo:400:kworker/u8:0.0 Request processed in 69us (978us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099992.000527:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1201
00010000:00080000:0.0:1766099992.000671:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000632c907e x1851885432962688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766099998 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766099992.000683:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432962688:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099992.000876:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000053dd58eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432962496:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099992.000912:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099992.000941:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766099992.000946:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006131d46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432962624:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099992.000968:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099992.001004:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432962560:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766099992.001037:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766099992.001066:0:5507:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962432
00000100:00100000:3.0:1766099992.001133:0:5507:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1875
00000100:00100000:3.0:1766099992.001137:0:5507:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432962432:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099992.001159:0:5507:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432962432:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (1544us total) trans 0 rc 0/0
00000100:00100000:3.0:1766099992.001166:0:5507:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1875
00000100:00100000:3.0:1766099992.001206:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065bd5068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432962432:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099992.001223:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962560
00000100:00100000:3.0:1766099992.001241:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1728
00000100:00100000:3.0:1766099992.001244:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b47533c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432962560:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766099992.001339:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766099992.001367:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b47533c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432962560:12345-0@lo:8:kworker/u8:0.0 Request processed in 122us (329us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766099992.001375:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1728
00000100:00080000:3.0:1766099992.001395:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766099992.001398:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766099992.001402:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432962560:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766099992.001506:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962496
00000100:00100000:0.0:1766099992.001527:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962624
00000100:00100000:0.0:1766099992.001544:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2769
00000100:00100000:0.0:1766099992.001548:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432962496:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099992.001584:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061093366 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432962496:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (672us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099992.001591:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2769
00000100:00100000:0.0:1766099992.001618:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2770
00000100:00100000:0.0:1766099992.001622:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432962624:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099992.001644:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055b9c8f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432962624:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (676us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099992.001650:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2770
00000100:00100000:0.0:1766099992.001675:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006131d46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432962624:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099992.001684:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000053dd58eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432962496:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766099992.573885:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099993.598197:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766099994.629622:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099995.647041:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099996.447230:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766099996.447279:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099996.447334:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239488
00000100:00100000:1.0:1766099996.447390:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2771
00000100:00100000:1.0:1766099996.447397:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417239488:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766099996.447440:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239424
00000100:00100000:2.0:1766099996.447973:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1876
00000100:00100000:2.0:1766099996.447978:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239424:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766099996.448926:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c2131aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417239488:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 1526us (1644us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099996.448936:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2771
00000100:00100000:3.0:1766099996.449106:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766099996.449468:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239552
00000100:00100000:1.0:1766099996.449521:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1729
00000100:00100000:1.0:1766099996.449526:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417239552:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766099996.449620:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766099996.449672:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417239552:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 145us (566us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766099996.449678:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1729
00000100:00100000:2.0:1766099996.450320:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239424:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2338us (3103us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099996.450330:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1876
00010000:00010000:0.0:1766099996.669747:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766099997.119195:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766099997.119958:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432962752:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099997.120038:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099997.120080:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962752
00000100:00100000:2.0:1766099997.120129:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1877
00000100:00100000:2.0:1766099997.120134:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432962752:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766099997.120144:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766099997.120149:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766099997.120152:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766099997.120156:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4500
00000100:00080000:0.0:1766099997.120158:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:0.0:1766099997.120162:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766099997.120182:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006ca3c93d x1851885432962880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766099997.120182:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432962752:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (146us total) trans 0 rc 0/0
00000100:00100000:2.0:1766099997.120189:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1877
00000100:00100000:1.0:1766099997.120290:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766099997.120299:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432963008:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099997.120331:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766099997.120358:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963008
00000100:00100000:1.0:1766099997.120383:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1202
00000100:00100000:1.0:1766099997.120387:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000053dd58eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432963008:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099997.120418:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000053dd58eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432963008:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (87us total) trans 0 rc 0/0
00000100:00100000:1.0:1766099997.120423:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1202
00010000:00080000:1.0:1766099997.120453:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000073b447f1 x1851885432963008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100004 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:1.0:1766099997.120464:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432963008:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099997.120896:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432962816:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766099997.120903:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432962880:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766099997.121040:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432962944:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766099997.121089:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432962752:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099997.121164:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766099997.121164:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099997.121195:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962816
00000100:00100000:0.0:1766099997.121245:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2772
00000100:00100000:0.0:1766099997.121249:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009296b030 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432962816:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099997.123046:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009296b030 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432962816:12345-0@lo:400:kworker/u8:0.0 Request processed in 1790us (1878us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099997.123060:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2772
00000100:00100000:0.0:1766099997.123124:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962880
00000100:00100000:0.0:1766099997.123162:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1730
00000100:00100000:0.0:1766099997.123167:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432962880:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766099997.123281:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766099997.123324:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432962880:12345-0@lo:8:kworker/u8:0.0 Request processed in 157us (2162us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766099997.123331:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1730
00000100:00100000:1.0:1766099997.123340:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766099997.123363:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432962944
00000100:00100000:0.0:1766099997.123386:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2773
00000100:00100000:0.0:1766099997.123389:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006131d46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432962944:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766099997.123434:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006131d46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432962944:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (95us total) trans 0 rc 0/0
00000100:00100000:0.0:1766099997.123441:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2773
00000100:00100000:2.0:1766099997.123532:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432962816:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766099997.123534:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432962944:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766099997.123562:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766099997.123566:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766099997.123569:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006ca3c93d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432962880:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766099997.694174:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099998.719034:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766099999.742384:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100000.765770:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100001.570517:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100001.570562:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100001.570641:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100001.570743:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239616
00000100:00100000:1.0:1766100001.570788:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1878
00000100:00100000:1.0:1766100001.570807:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239616:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100001.570875:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417239616:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 69us (366us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100001.570883:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1878
00000100:00100000:0.0:1766100001.572393:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239680
00000100:00100000:0.0:1766100001.572441:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2774
00000100:00100000:0.0:1766100001.572447:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417239680:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100001.572517:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065bd5068 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417239680:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 69us (1955us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100001.572523:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2774
00000100:00100000:2.0:1766100001.573658:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239744
00000100:00100000:2.0:1766100001.573696:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1731
00000100:00100000:2.0:1766100001.573701:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417239744:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766100001.573806:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100001.573868:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417239744:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 165us (3227us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100001.573875:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1731
00010000:00010000:0.0:1766100001.789689:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100002.237915:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100002.237923:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100002.237926:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100002.237934:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4505
00000100:00080000:1.0:1766100002.237938:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00100000:2.0:1766100002.237943:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432963136:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100002.237944:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100002.238016:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100002.238036:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c56d4390 x1851885432963200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100002.238079:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432963200:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100002.238112:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963136
00000100:00100000:2.0:1766100002.238123:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100002.238146:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a846069e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432963264:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100002.238163:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2775
00000100:00100000:2.0:1766100002.238170:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963200
00000100:00100000:1.0:1766100002.238171:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432963136:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100002.238183:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100002.238198:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1732
00000100:00100000:2.0:1766100002.238204:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432963200:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100002.238222:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963264
00000100:00100000:1.0:1766100002.238242:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e2ba551 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432963136:12345-0@lo:400:kworker/u8:3.0 Request processed in 65us (220us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100002.238248:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2776
00000100:00100000:0.0:1766100002.238255:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432963264:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100002.238258:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2775
00010000:02020000:2.0:1766100002.238294:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100002.238295:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006ca3c93d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432963264:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (112us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100002.238315:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2776
00000100:00100000:1.0:1766100002.238320:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a846069e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432963264:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100002.238342:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432963200:12345-0@lo:8:kworker/u8:3.0 Request processed in 135us (219us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100002.238349:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1732
00000100:00080000:2.0:1766100002.238375:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100002.238379:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100002.238382:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432963200:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100002.238400:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432963136:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100002.238678:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432963328:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100002.238710:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100002.238734:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100002.238738:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432963072:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100002.238759:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100002.238966:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963328
00000100:00100000:2.0:1766100002.238997:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1203
00000100:00100000:2.0:1766100002.239001:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432963328:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100002.239045:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432963328:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (334us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100002.239051:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1203
00010000:00080000:3.0:1766100002.239113:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009ecb9adb x1851885432963328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100009 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100002.239123:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432963328:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100002.239410:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963072
00000100:00100000:1.0:1766100002.239449:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1879
00000100:00100000:1.0:1766100002.239454:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432963072:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100002.239523:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432963072:12345-0@lo:400:kworker/u8:3.0 Request processed in 68us (764us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100002.239530:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1879
00000100:00100000:2.0:1766100002.240588:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432963072:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100002.814746:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100003.837689:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100004.861676:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100005.885814:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100006.685024:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100006.685072:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100006.685117:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100006.685155:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239936
00000100:00100000:1.0:1766100006.685189:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1733
00000100:00100000:1.0:1766100006.685195:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417239936:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100006.685297:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100006.685352:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417239936:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 156us (236us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100006.685360:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1733
00000100:00100000:3.0:1766100006.685572:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239808
00000100:00100000:3.0:1766100006.685614:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1880
00000100:00100000:3.0:1766100006.685618:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417239808:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100006.685650:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417239808:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 32us (634us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100006.685656:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1880
00000100:00100000:2.0:1766100006.685891:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417239872
00000100:00100000:2.0:1766100006.685929:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2777
00000100:00100000:2.0:1766100006.685933:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a846069e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417239872:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100006.685979:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a846069e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417239872:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 45us (907us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100006.685985:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2777
00010000:00010000:1.0:1766100006.910654:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100007.357849:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100007.357857:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100007.357860:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100007.357867:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4510
00000100:00080000:0.0:1766100007.357871:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100007.357876:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100007.357897:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000003dd022ac x1851885432963520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100007.357990:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432963648:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358060:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100007.358103:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766100007.358109:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432963392:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358156:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100007.358182:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432963584:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358208:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100007.358225:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100007.358231:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432963456:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358252:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100007.358283:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963392
00000100:00100000:1.0:1766100007.358319:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1881
00000100:00100000:1.0:1766100007.358325:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432963392:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358368:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432963392:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (213us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100007.358376:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1881
00000100:00100000:1.0:1766100007.358407:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963584
00000100:00100000:1.0:1766100007.358431:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963456
00000100:00100000:1.0:1766100007.358448:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2778
00000100:00100000:1.0:1766100007.358452:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000677df168 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432963584:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358506:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000677df168 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432963584:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (298us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100007.358512:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2778
00000100:00100000:1.0:1766100007.358524:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2779
00000100:00100000:1.0:1766100007.358527:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9f903ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432963456:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358553:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9f903ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432963456:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (300us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100007.358558:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2779
00000100:00100000:1.0:1766100007.358599:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432963456:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358619:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073b447f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432963584:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358647:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bf7662d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432963392:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358679:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432963520:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358715:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100007.358768:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963648
00000100:00100000:3.0:1766100007.358880:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963520
00000100:00100000:3.0:1766100007.358904:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1734
00000100:00100000:3.0:1766100007.358907:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432963520:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100007.358962:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1204
00000100:00100000:1.0:1766100007.358966:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a951f581 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432963648:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100007.359004:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a951f581 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432963648:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (946us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100007.359011:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1204
00010000:02020000:3.0:1766100007.359305:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100007.359345:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432963520:12345-0@lo:8:kworker/u8:0.0 Request processed in 436us (629us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100007.359352:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1734
00000100:00080000:3.0:1766100007.359382:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100007.359386:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100007.359389:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432963520:0@lo:8:kworker/u8:0.0
00010000:00080000:3.0:1766100007.359441:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005dcf37af x1851885432963648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100014 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766100007.361231:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dcf37af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432963648:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766100007.936050:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100008.957799:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100009.982250:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100011.006220:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100011.805686:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100011.806253:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100011.806443:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240000
00000100:00100000:0.0:1766100011.806550:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1882
00000100:00100000:0.0:1766100011.806558:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417240000:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100011.806663:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417240000:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 105us (411us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100011.806672:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1882
00000100:00100000:3.0:1766100011.807078:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240064
00000100:00100000:3.0:1766100011.807127:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2780
00000100:00100000:3.0:1766100011.807132:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4d173d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417240064:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100011.807172:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4d173d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417240064:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 39us (1500us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100011.807179:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2780
00000100:00100000:3.0:1766100011.809089:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100011.809192:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240128
00000100:00100000:1.0:1766100011.809245:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1735
00000100:00100000:1.0:1766100011.809254:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018d302a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240128:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100011.809709:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100011.809814:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018d302a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240128:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 559us (735us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100011.809823:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1735
00010000:00010000:0.0:1766100012.030565:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100012.477913:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100012.477921:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100012.477925:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100012.477935:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4515
00000100:00080000:2.0:1766100012.477939:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100012.477955:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100012.477982:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e76b9049 x1851885432963840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100012.478032:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432963776:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.478113:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100012.478154:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100012.478159:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a65af3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432963904:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.478177:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100012.478198:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e76b9049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432963840:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100012.478221:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100012.478256:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963776
00000100:00100000:2.0:1766100012.478299:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963904
00000100:00100000:2.0:1766100012.478315:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2781
00000100:00100000:2.0:1766100012.478320:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057c568c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432963776:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.478369:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057c568c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432963776:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (258us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100012.478375:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2781
00000100:00100000:2.0:1766100012.478393:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2782
00000100:00100000:2.0:1766100012.478396:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432963904:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.478414:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086644dcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432963904:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (237us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100012.478419:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2782
00000100:00100000:2.0:1766100012.478450:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a65af3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432963904:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.478458:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432963776:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100012.478683:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e68d47de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432963968:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100012.478739:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100012.478765:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100012.478769:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432963712:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100012.478793:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100012.478819:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963712
00000100:00100000:0.0:1766100012.478841:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1883
00000100:00100000:0.0:1766100012.478845:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432963712:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100012.478869:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432963712:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (76us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100012.478874:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1883
00000100:00100000:0.0:1766100012.478904:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8b37342 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432963712:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.479464:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963840
00000100:00100000:2.0:1766100012.479509:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432963968
00000100:00100000:2.0:1766100012.479537:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1205
00000100:00100000:2.0:1766100012.479542:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce45f83d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432963968:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100012.479580:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce45f83d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432963968:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (841us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100012.479585:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1205
00000100:00100000:2.0:1766100012.479595:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1736
00000100:00100000:2.0:1766100012.479598:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432963840:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766100012.479706:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100012.479744:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432963840:12345-0@lo:8:kworker/u8:0.0 Request processed in 144us (1522us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100012.479750:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1736
00000100:00080000:2.0:1766100012.479776:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100012.479780:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100012.479784:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e76b9049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432963840:0@lo:8:kworker/u8:0.0
00010000:00080000:0.0:1766100012.479851:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e68d47de x1851885432963968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100019 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766100012.479861:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e68d47de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432963968:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766100013.055209:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100014.077693:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100015.101835:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100016.127718:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100016.924704:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100016.924756:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100016.924801:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100016.924886:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240256
00000100:00100000:1.0:1766100016.924968:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2783
00000100:00100000:1.0:1766100016.924976:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e76b9049 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417240256:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100016.925076:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e76b9049 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417240256:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 100us (381us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100016.925083:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2783
00000100:00100000:0.0:1766100016.925388:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240192
00000100:00100000:0.0:1766100016.925448:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1884
00000100:00100000:0.0:1766100016.925455:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417240192:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100016.925553:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417240192:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 97us (797us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100016.925561:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1884
00000100:00100000:3.0:1766100016.925801:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240320
00000100:00100000:3.0:1766100016.925837:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1737
00000100:00100000:3.0:1766100016.925842:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417240320:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100016.925936:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100016.925980:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417240320:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 136us (1178us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100016.925987:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1737
00010000:00010000:0.0:1766100017.150308:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100017.598356:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100017.598363:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100017.598366:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100017.598373:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4520
00000100:00080000:1.0:1766100017.598376:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100017.598382:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100017.598401:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000046cfd9d1 x1851885432964160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100017.598440:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432964224:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100017.598516:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100017.598542:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100017.598547:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432964096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100017.598564:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100017.598590:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964224
00000100:00100000:1.0:1766100017.598627:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964096
00000100:00100000:1.0:1766100017.598644:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2784
00000100:00100000:1.0:1766100017.598650:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432964224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100017.598703:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aaed12d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432964224:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (188us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100017.598711:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2784
00000100:00100000:1.0:1766100017.598726:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2785
00000100:00100000:1.0:1766100017.598729:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432964096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100017.598749:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092cf9228 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432964096:12345-0@lo:400:kworker/u8:3.0 Request processed in 19us (185us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100017.598754:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2785
00000100:00100000:1.0:1766100017.598791:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432964096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100017.598799:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432964224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100017.598868:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009bbfcf2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432964288:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100017.598898:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100017.598923:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100017.598928:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432964032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100017.598953:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100017.598975:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964032
00000100:00100000:2.0:1766100017.598997:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1885
00000100:00100000:2.0:1766100017.599000:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432964032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100017.599034:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432964032:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (82us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100017.599040:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1885
00000100:00100000:2.0:1766100017.599065:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432964160:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100017.599087:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100017.599110:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964288
00000100:00100000:2.0:1766100017.599129:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964160
00000100:00100000:2.0:1766100017.599141:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1206
00000100:00100000:2.0:1766100017.599145:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432964288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100017.599167:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432964288:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (269us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100017.599173:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1206
00000100:00100000:2.0:1766100017.599181:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1738
00000100:00100000:2.0:1766100017.599184:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021448680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432964160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100017.599278:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100017.599301:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021448680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432964160:12345-0@lo:8:kworker/u8:3.0 Request processed in 116us (214us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100017.599306:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1738
00000100:00100000:2.0:1766100017.599425:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092653886 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432964032:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766100017.599433:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009bbfcf2c x1851885432964288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100024 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100017.599441:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009bbfcf2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432964288:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766100017.599459:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100017.599462:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100017.599466:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432964160:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766100018.173640:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100019.197937:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100020.222001:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100021.245705:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100022.048747:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100022.049315:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100022.049419:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240448
00000100:00100000:1.0:1766100022.049505:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2786
00000100:00100000:1.0:1766100022.049512:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041477718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417240448:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100022.049598:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041477718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417240448:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 86us (284us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100022.049606:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2786
00000100:00100000:2.0:1766100022.050693:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240384
00000100:00100000:2.0:1766100022.050740:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1886
00000100:00100000:2.0:1766100022.050745:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417240384:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100022.050802:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417240384:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 55us (2063us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100022.050809:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1886
00000100:00100000:1.0:1766100022.052078:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100022.052379:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240512
00000100:00100000:0.0:1766100022.052549:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1739
00000100:00100000:0.0:1766100022.052555:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240512:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766100022.052661:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100022.052725:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240512:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 168us (652us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100022.052733:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1739
00010000:00010000:1.0:1766100022.270073:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100022.718255:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100022.718262:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100022.718266:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100022.718274:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4526
00000100:00080000:1.0:1766100022.718277:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100022.718282:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100022.718301:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000066defd5f x1851885432964480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100022.718345:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432964416:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100022.718409:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100022.718441:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100022.718446:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432964544:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100022.718467:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100022.718520:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964416
00000100:00100000:1.0:1766100022.718555:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964544
00000100:00100000:1.0:1766100022.718571:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2787
00000100:00100000:1.0:1766100022.718575:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009bbfcf2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432964416:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100022.718619:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009bbfcf2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432964416:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (211us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100022.718626:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2787
00000100:00100000:3.0:1766100022.718702:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432964608:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.718737:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100022.718756:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100022.718761:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432964352:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.718786:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100022.718808:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964608
00000100:00100000:3.0:1766100022.718830:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1207
00000100:00100000:3.0:1766100022.718834:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432964608:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.718866:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432964608:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (130us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100022.718873:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1207
00000100:00100000:3.0:1766100022.718892:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964352
00000100:00100000:3.0:1766100022.718912:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1887
00000100:00100000:3.0:1766100022.718916:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432964352:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.718940:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432964352:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (154us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100022.718945:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1887
00000100:00100000:3.0:1766100022.718980:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432964352:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766100022.718990:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000046cfd9d1 x1851885432964608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100029 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766100022.718999:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432964608:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.719039:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2788
00000100:00100000:3.0:1766100022.719043:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432964544:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.719070:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092653886 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432964544:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (603us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100022.719075:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2788
00000100:00100000:3.0:1766100022.719118:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004584cb44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432964544:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100022.719127:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432964416:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100022.719293:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432964480:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100022.719322:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100022.719360:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964480
00000100:00100000:2.0:1766100022.719380:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1740
00000100:00100000:2.0:1766100022.719384:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432964480:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766100022.719503:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100022.719537:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432964480:12345-0@lo:8:kworker/u8:0.0 Request processed in 152us (215us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100022.719544:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1740
00000100:00080000:2.0:1766100022.719568:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100022.719572:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100022.719576:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000066defd5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432964480:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766100023.293800:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100024.319044:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100025.341992:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100026.365873:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100027.165395:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100027.167335:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240576
00000100:00100000:2.0:1766100027.167425:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1888
00000100:00100000:2.0:1766100027.167434:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417240576:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100027.167569:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417240576:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 134us (2186us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100027.167582:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1888
00000100:00100000:0.0:1766100027.168517:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100027.168565:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100027.168843:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240704
00000100:00100000:3.0:1766100027.168891:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1741
00000100:00100000:3.0:1766100027.168897:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240704:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100027.169008:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100027.169089:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240704:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 191us (523us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100027.169099:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1741
00000100:00100000:1.0:1766100027.169519:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240640
00000100:00100000:1.0:1766100027.169569:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2789
00000100:00100000:1.0:1766100027.169574:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce45f83d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417240640:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100027.170827:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce45f83d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417240640:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1242us (2307us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100027.170842:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2789
00010000:00010000:0.0:1766100027.389797:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100027.841303:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100027.841316:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100027.841321:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100027.841332:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4531
00000100:00080000:1.0:1766100027.841336:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100027.841342:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100027.841343:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432964736:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100027.841368:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a951f581 x1851885432964800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100027.841421:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432964864:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100027.841432:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100027.841546:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432964672:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100027.841554:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100027.841624:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964864
00000100:00100000:1.0:1766100027.841624:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964736
00000100:00100000:0.0:1766100027.841680:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2790
00000100:00100000:0.0:1766100027.841689:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432964864:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100027.841781:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2791
00000100:00100000:2.0:1766100027.841783:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432964928:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100027.841789:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000337df21d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432964736:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100027.841893:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100027.843519:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100027.843645:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964928
00000100:00100000:0.0:1766100027.843689:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066defd5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432964864:12345-0@lo:400:kworker/u8:3.0 Request processed in 1990us (2129us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100027.843708:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2790
00000100:00100000:0.0:1766100027.843775:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432964800:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100027.843790:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964672
00000100:00100000:3.0:1766100027.843850:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1889
00000100:00100000:0.0:1766100027.843854:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100027.843862:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432964672:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100027.843875:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964800
00000100:00100000:2.0:1766100027.843938:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1208
00000100:00100000:2.0:1766100027.843945:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432964928:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100027.843948:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432964672:12345-0@lo:400:kworker/u8:3.0 Request processed in 88us (2055us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100027.843957:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1889
00000100:00100000:2.0:1766100027.844931:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432964928:12345-0@lo:400:kworker/u8:3.0 Request processed in 982us (1414us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100027.844943:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1208
00000100:00100000:2.0:1766100027.844963:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1742
00000100:00100000:2.0:1766100027.844968:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432964800:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100027.845185:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100027.845266:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432964800:12345-0@lo:8:kworker/u8:3.0 Request processed in 297us (1420us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100027.845273:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1742
00010000:00080000:2.0:1766100027.845368:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c56d4390 x1851885432964928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100034 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100027.845397:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432964928:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100027.845546:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100027.845555:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100027.845562:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432964800:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100027.845644:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432964864:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100027.845651:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000337df21d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432964736:12345-0@lo:400:kworker/u8:3.0 Request processed in 3857us (4214us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100027.845668:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2791
00000100:00100000:0.0:1766100027.845710:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432964736:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100027.845776:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432964672:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100028.414143:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100029.445689:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100030.463298:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100031.487511:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100032.284626:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100032.284671:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100032.284726:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100032.284828:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240896
00000100:00100000:3.0:1766100032.284876:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1743
00000100:00100000:3.0:1766100032.284883:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240896:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100032.284981:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100032.285042:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417240896:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 158us (425us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100032.285050:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1743
00000100:00100000:2.0:1766100032.286685:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240832
00000100:00100000:2.0:1766100032.286737:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2792
00000100:00100000:2.0:1766100032.286743:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417240832:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100032.286809:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004584cb44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417240832:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 65us (2138us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100032.286817:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2792
00000100:00100000:0.0:1766100032.286888:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240768
00000100:00100000:0.0:1766100032.286922:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1890
00000100:00100000:0.0:1766100032.286927:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417240768:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100032.286967:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417240768:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 40us (2242us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100032.286974:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1890
00010000:00010000:0.0:1766100032.511526:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100032.957825:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100032.957833:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100032.957836:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100032.957845:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4536
00000100:00080000:1.0:1766100032.957848:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100032.957853:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100032.957874:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f7427f30 x1851885432965120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100032.958030:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432965056:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958091:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100032.958123:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100032.958128:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432965184:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958147:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100032.958172:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432965120:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958195:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100032.958238:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965056
00000100:00100000:0.0:1766100032.958275:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965184
00000100:00100000:0.0:1766100032.958294:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2793
00000100:00100000:0.0:1766100032.958299:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000234e8a5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432965056:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958344:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000234e8a5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432965056:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (252us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100032.958351:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2793
00000100:00100000:0.0:1766100032.958366:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2794
00000100:00100000:0.0:1766100032.958369:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009c235b45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432965184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958390:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009c235b45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432965184:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (242us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100032.958396:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2794
00000100:00100000:0.0:1766100032.958418:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965120
00000100:00100000:0.0:1766100032.958435:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1744
00000100:00100000:0.0:1766100032.958438:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432965120:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100032.958545:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100032.958585:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432965120:12345-0@lo:8:kworker/u8:3.0 Request processed in 146us (389us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100032.958592:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1744
00000100:00080000:0.0:1766100032.958622:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100032.958625:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100032.958629:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432965120:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958657:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432965184:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.958665:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432965056:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100032.958761:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432964992:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100032.958794:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100032.958823:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100032.958827:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432965248:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100032.958850:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100032.958871:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965248
00000100:00100000:3.0:1766100032.958891:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1209
00000100:00100000:3.0:1766100032.958894:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432965248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100032.958923:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432965248:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (73us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100032.958928:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1209
00010000:00080000:3.0:1766100032.958949:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009ecb9adb x1851885432965248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100039 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100032.958957:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432965248:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.961268:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432964992
00000100:00100000:0.0:1766100032.961324:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1891
00000100:00100000:0.0:1766100032.961330:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432964992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100032.961412:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432964992:12345-0@lo:400:kworker/u8:3.0 Request processed in 81us (2617us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100032.961418:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1891
00000100:00100000:3.0:1766100032.961540:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432964992:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100033.533709:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100034.558225:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100035.585503:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100036.606362:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100037.410772:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100037.410829:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100037.411459:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100037.411618:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241024
00000100:00100000:2.0:1766100037.411671:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2795
00000100:00100000:2.0:1766100037.411677:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018d302a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241024:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100037.411743:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018d302a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241024:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 66us (979us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100037.411750:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2795
00000100:00100000:0.0:1766100037.414423:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417240960
00000100:00100000:0.0:1766100037.414501:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1892
00000100:00100000:0.0:1766100037.414506:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417240960:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100037.414589:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417240960:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 82us (3760us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100037.414596:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1892
00000100:00100000:1.0:1766100037.415210:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241088
00000100:00100000:1.0:1766100037.415247:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1745
00000100:00100000:1.0:1766100037.415253:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000923c0bd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417241088:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766100037.415345:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100037.415387:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000923c0bd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417241088:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 134us (3930us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100037.415395:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1745
00010000:00010000:1.0:1766100037.631709:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100038.077699:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100038.077706:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100038.077709:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100038.077717:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4541
00000100:00080000:1.0:1766100038.077720:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100038.077726:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100038.077748:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000046cfd9d1 x1851885432965440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100038.077870:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432965312:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100038.077941:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100038.077979:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100038.077984:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432965568:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100038.078013:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100038.078045:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965568
00000100:00100000:3.0:1766100038.078082:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1210
00000100:00100000:3.0:1766100038.078088:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027be1d19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432965568:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100038.078133:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027be1d19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432965568:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (120us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100038.078140:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1210
00010000:00080000:3.0:1766100038.078190:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005063cee5 x1851885432965568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100045 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766100038.078201:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005063cee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432965568:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100038.078414:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965312
00000100:00100000:1.0:1766100038.078451:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1893
00000100:00100000:1.0:1766100038.078454:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432965312:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100038.078510:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432965312:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (569us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100038.078516:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1893
00000100:00100000:3.0:1766100038.078659:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432965312:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100038.078757:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432965376:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100038.078790:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100038.078816:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100038.078822:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432965504:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100038.078842:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100038.078942:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432965440:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100038.078967:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100038.078993:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965440
00000100:00100000:1.0:1766100038.079011:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1746
00000100:00100000:1.0:1766100038.079014:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432965440:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766100038.079105:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100038.079135:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432965440:12345-0@lo:8:kworker/u8:0.0 Request processed in 120us (168us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100038.079144:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1746
00000100:00080000:1.0:1766100038.079178:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100038.079181:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100038.079185:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432965440:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100038.080006:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965376
00000100:00100000:2.0:1766100038.080040:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965504
00000100:00100000:2.0:1766100038.080060:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2796
00000100:00100000:2.0:1766100038.080064:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb7aab30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432965376:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100038.080115:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb7aab30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432965376:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (1325us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100038.080121:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2796
00000100:00100000:2.0:1766100038.080135:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2797
00000100:00100000:2.0:1766100038.080138:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a60be776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432965504:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100038.080176:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a60be776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432965504:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (1319us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100038.080181:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2797
00000100:00100000:2.0:1766100038.080219:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432965504:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100038.080229:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432965376:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766100038.653707:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100039.678061:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100040.704012:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100041.725769:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100042.530245:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100042.530298:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100042.530350:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100042.530464:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241152
00000100:00100000:1.0:1766100042.530560:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1894
00000100:00100000:1.0:1766100042.530566:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417241152:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100042.530633:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417241152:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 67us (335us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100042.530640:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1894
00000100:00100000:3.0:1766100042.530764:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241216
00000100:00100000:3.0:1766100042.530805:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2798
00000100:00100000:3.0:1766100042.530810:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c450cdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241216:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100042.530843:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c450cdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241216:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 32us (605us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100042.530850:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2798
00000100:00100000:2.0:1766100042.530958:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241280
00000100:00100000:2.0:1766100042.531152:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1747
00000100:00100000:2.0:1766100042.531157:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066062a18 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417241280:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766100042.531248:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100042.531279:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066062a18 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417241280:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 121us (929us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100042.531285:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1747
00010000:00010000:0.0:1766100042.751199:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100043.197865:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100043.197873:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100043.197876:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100043.197885:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4546
00000100:00080000:3.0:1766100043.197889:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100043.197896:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100043.197918:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000084112edb x1851885432965760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100043.197969:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003a6badc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432965888:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198032:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100043.198059:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100043.198065:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008da07da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432965632:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198097:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100043.198122:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c4f798bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432965824:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100043.198142:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965632
00000100:00100000:3.0:1766100043.198151:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100043.198169:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766100043.198175:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000edd8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432965696:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198193:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100043.198201:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1895
00000100:00100000:0.0:1766100043.198208:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432965632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198217:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084112edb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432965760:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198242:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100043.198272:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432965632:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (173us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100043.198276:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965888
00000100:00100000:0.0:1766100043.198283:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1895
00000100:00100000:3.0:1766100043.198302:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965760
00000100:00100000:0.0:1766100043.198320:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965824
00000100:00100000:3.0:1766100043.198325:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1211
00000100:00100000:3.0:1766100043.198331:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7b8f472 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432965888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100043.198352:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965696
00000100:00100000:3.0:1766100043.198364:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7b8f472 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432965888:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (333us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100043.198371:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1211
00000100:00100000:0.0:1766100043.198375:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2799
00000100:00100000:0.0:1766100043.198380:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000305829f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432965824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198389:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1748
00000100:00100000:3.0:1766100043.198393:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ec0c7a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432965760:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100043.198419:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000305829f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432965824:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (269us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100043.198426:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2799
00000100:00100000:0.0:1766100043.198439:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2800
00000100:00100000:0.0:1766100043.198443:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3831caa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432965696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100043.198467:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3831caa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432965696:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (274us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100043.198472:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2800
00010000:02020000:3.0:1766100043.198528:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100043.198551:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000edd8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432965696:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198566:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ec0c7a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432965760:12345-0@lo:8:kworker/u8:3.0 Request processed in 171us (324us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100043.198567:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c4f798bf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432965824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100043.198579:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1748
00000100:00100000:0.0:1766100043.198607:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008da07da1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432965632:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766100043.198618:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00010000:00080000:0.0:1766100043.198622:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000003a6badc x1851885432965888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100050 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00080000:3.0:1766100043.198623:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100043.198630:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084112edb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432965760:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100043.198637:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003a6badc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432965888:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100043.773675:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100044.797805:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100045.821785:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100046.848121:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100047.645212:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100047.645262:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100047.645350:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241408
00000100:00100000:2.0:1766100047.645410:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2801
00000100:00100000:2.0:1766100047.645416:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d99686c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241408:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100047.645559:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d99686c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241408:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 142us (354us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100047.645566:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2801
00000100:00100000:1.0:1766100047.645887:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241344
00000100:00100000:2.0:1766100047.645997:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100047.646066:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241472
00000100:00100000:3.0:1766100047.646099:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1749
00000100:00100000:3.0:1766100047.646103:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417241472:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100047.646191:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100047.646232:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417241472:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 127us (235us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100047.646239:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1749
00000100:00100000:1.0:1766100047.646692:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1896
00000100:00100000:1.0:1766100047.646698:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417241344:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100047.647903:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417241344:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1200us (2637us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100047.647914:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1896
00010000:00010000:0.0:1766100047.869808:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100048.318237:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100048.318245:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100048.318248:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100048.318255:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4551
00000100:00080000:2.0:1766100048.318258:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100048.318263:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100048.318283:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a8a851b9 x1851885432966080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100048.318334:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e68d47de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432966016:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100048.318398:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100048.318423:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100048.318429:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2734238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432966144:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100048.318466:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100048.318530:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966016
00000100:00100000:2.0:1766100048.318565:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966144
00000100:00100000:2.0:1766100048.318586:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2802
00000100:00100000:2.0:1766100048.318592:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005001f4c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432966016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100048.318641:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005001f4c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432966016:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (244us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100048.318648:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2802
00000100:00100000:2.0:1766100048.318664:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2803
00000100:00100000:2.0:1766100048.318668:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432966144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100048.318694:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c94477ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432966144:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (228us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100048.318700:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2803
00000100:00100000:2.0:1766100048.318743:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2734238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432966144:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100048.318755:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e68d47de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432966016:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100048.319277:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a55df54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432966208:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100048.319315:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100048.319348:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100048.319354:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432965952:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100048.319379:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100048.319683:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a851b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432966080:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100048.319714:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100048.320317:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966208
00000100:00100000:0.0:1766100048.320352:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966080
00000100:00100000:0.0:1766100048.320370:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1212
00000100:00100000:0.0:1766100048.320375:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432966208:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100048.320425:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432966208:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (1109us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100048.320431:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1212
00000100:00100000:0.0:1766100048.320458:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1750
00000100:00100000:0.0:1766100048.320462:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432966080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100048.320579:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100048.320615:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432966080:12345-0@lo:8:kworker/u8:3.0 Request processed in 151us (901us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100048.320621:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1750
00000100:00080000:0.0:1766100048.320657:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100048.320662:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100048.320666:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a851b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432966080:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766100048.320695:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000007a55df54 x1851885432966208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100055 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100048.320710:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a55df54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432966208:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100048.324033:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432965952
00000100:00100000:1.0:1766100048.324096:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1897
00000100:00100000:1.0:1766100048.324103:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432965952:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100048.324219:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432965952:12345-0@lo:400:kworker/u8:3.0 Request processed in 116us (4839us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100048.324230:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1897
00000100:00100000:3.0:1766100048.327548:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec6e3728 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432965952:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100048.893787:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100049.923685:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100050.942530:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100051.965775:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100052.767062:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100052.767117:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100052.767145:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241600
00000100:00100000:2.0:1766100052.767158:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100052.767192:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241664
00000100:00100000:0.0:1766100052.767199:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2804
00000100:00100000:0.0:1766100052.767208:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2734238 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417241600:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100052.767221:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1751
00000100:00100000:2.0:1766100052.767227:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417241664:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00000100:00100000:0.0:1766100052.767281:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2734238 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417241600:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 72us (227us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100052.767289:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2804
00010000:02020000:2.0:1766100052.767390:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100052.767446:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417241664:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 219us (288us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100052.767453:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1751
00000100:00100000:1.0:1766100052.768962:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241536
00000100:00100000:1.0:1766100052.769016:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1898
00000100:00100000:1.0:1766100052.769021:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417241536:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100052.769086:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417241536:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 64us (1970us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100052.769093:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1898
00010000:00010000:1.0:1766100052.990260:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100053.438561:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100053.438568:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100053.438571:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100053.438578:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4556
00000100:00080000:0.0:1766100053.438581:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100053.438587:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100053.438605:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002b836d1e x1851885432966400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100053.438645:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b836d1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432966400:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100053.438703:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100053.438851:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043db2b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432966272:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100053.438886:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100053.438902:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100053.438907:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003520602a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432966528:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100053.438926:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100053.438998:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061acc5af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432966336:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100053.439028:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100053.439055:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100053.439060:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000838122d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432966464:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100053.439077:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100053.439107:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966400
00000100:00100000:2.0:1766100053.439131:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966528
00000100:00100000:2.0:1766100053.439154:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1213
00000100:00100000:2.0:1766100053.439159:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432966528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100053.439205:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432966528:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (278us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100053.439213:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1213
00000100:00100000:2.0:1766100053.439228:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1752
00000100:00100000:2.0:1766100053.439232:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432966400:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100053.439338:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100053.439365:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432966400:12345-0@lo:8:kworker/u8:3.0 Request processed in 133us (664us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100053.439371:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1752
00000100:00100000:2.0:1766100053.439388:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966272
00000100:00100000:2.0:1766100053.439407:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1899
00000100:00100000:2.0:1766100053.439410:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432966272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100053.439450:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432966272:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (564us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100053.439457:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1899
00010000:00080000:2.0:1766100053.439514:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003520602a x1851885432966528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100060 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100053.439530:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003520602a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432966528:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100053.439539:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043db2b7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432966272:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100053.439690:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966336
00000100:00100000:3.0:1766100053.439726:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966464
00000100:00100000:3.0:1766100053.439744:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2805
00000100:00100000:3.0:1766100053.439748:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432966336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100053.439783:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c97d50d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432966336:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (754us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100053.439789:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2805
00000100:00100000:3.0:1766100053.439801:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2806
00000100:00100000:3.0:1766100053.439804:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432966464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100053.439828:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007f24045 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432966464:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (750us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100053.439834:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2806
00000100:00100000:2.0:1766100053.439974:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000838122d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432966464:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100053.439984:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061acc5af pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432966336:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100053.440130:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100053.440135:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100053.440139:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b836d1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432966400:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766100054.015872:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100055.037813:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100056.065629:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100057.085902:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100057.885021:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100057.885080:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100057.885185:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241856
00000100:00100000:0.0:1766100057.885242:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1753
00000100:00100000:0.0:1766100057.885249:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417241856:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766100057.885346:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100057.885423:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417241856:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 173us (410us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100057.885431:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1753
00000100:00100000:1.0:1766100057.885508:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241728
00000100:00100000:1.0:1766100057.885541:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1900
00000100:00100000:1.0:1766100057.885545:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417241728:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100057.885581:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417241728:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 35us (501us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100057.885588:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1900
00000100:00100000:3.0:1766100057.888787:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100057.888885:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241792
00000100:00100000:0.0:1766100057.888954:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2807
00000100:00100000:0.0:1766100057.888961:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241792:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100057.889048:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6c2b760 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417241792:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 86us (269us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100057.889057:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2807
00010000:00010000:0.0:1766100058.109668:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100058.558554:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100058.558559:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100058.558562:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100058.558569:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4561
00000100:00080000:1.0:1766100058.558571:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100058.558576:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100058.558593:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c56d4390 x1851885432966720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100058.558631:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432966656:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.558686:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100058.558713:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100058.558717:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432966784:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.558733:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100058.558761:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966656
00000100:00100000:1.0:1766100058.558793:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966784
00000100:00100000:1.0:1766100058.558806:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2808
00000100:00100000:1.0:1766100058.558811:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432966656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.558853:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005063cee5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432966656:12345-0@lo:400:kworker/u8:3.0 Request processed in 41us (167us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100058.558859:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2808
00000100:00100000:1.0:1766100058.558874:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2809
00000100:00100000:1.0:1766100058.558877:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432966784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.558904:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000057bb2ef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432966784:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (170us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100058.558910:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2809
00000100:00100000:3.0:1766100058.558924:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432966592:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.558950:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432966784:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100058.558962:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100058.558966:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432966656:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100058.558987:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100058.558994:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432966848:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.558997:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432966720:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100058.559029:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100058.559059:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966592
00000100:00100000:3.0:1766100058.559068:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966848
00000100:00100000:1.0:1766100058.559087:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1901
00000100:00100000:1.0:1766100058.559092:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432966592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100058.559104:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1214
00000100:00100000:3.0:1766100058.559110:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432966848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.559125:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432966592:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (164us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100058.559134:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1901
00000100:00100000:3.0:1766100058.559150:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432966848:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (121us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100058.559157:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1214
00000100:00100000:1.0:1766100058.559173:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:3.0:1766100058.559190:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a951f581 x1851885432966848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100065 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100058.559200:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a951f581 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432966848:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.559204:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966720
00000100:00100000:3.0:1766100058.559211:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046cfd9d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432966592:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100058.559224:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1754
00000100:00100000:1.0:1766100058.559227:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432966720:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100058.559326:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100058.559366:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432966720:12345-0@lo:8:kworker/u8:3.0 Request processed in 137us (194us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100058.559372:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1754
00000100:00080000:3.0:1766100058.559586:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100058.559590:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100058.559594:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c56d4390 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432966720:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766100059.136361:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100060.157853:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100061.183095:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100062.205792:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100063.005228:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100063.005941:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100063.006740:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241920
00000100:00100000:2.0:1766100063.006799:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1902
00000100:00100000:2.0:1766100063.006805:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417241920:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100063.006884:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417241920:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 78us (944us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100063.006891:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1902
00000100:00100000:1.0:1766100063.007552:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417241984
00000100:00100000:1.0:1766100063.007589:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2810
00000100:00100000:1.0:1766100063.007593:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417241984:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100063.007624:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064d7ca73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417241984:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 31us (2408us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100063.007630:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2810
00000100:00100000:3.0:1766100063.008067:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100063.010376:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242048
00000100:00100000:1.0:1766100063.010455:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1755
00000100:00100000:1.0:1766100063.010463:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417242048:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100063.010577:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100063.010653:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417242048:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 191us (2588us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100063.010660:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1755
00010000:00010000:0.0:1766100063.229828:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100063.677755:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100063.677762:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100063.677766:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100063.677773:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4567
00000100:00080000:1.0:1766100063.677776:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100063.677782:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100063.677806:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006e177295 x1851885432967040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100063.677859:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432967104:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100063.677929:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100063.677963:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100063.677968:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432966976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100063.677988:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100063.678010:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432967040:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100063.678033:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100063.678067:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967104
00000100:00100000:1.0:1766100063.678112:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966976
00000100:00100000:1.0:1766100063.678132:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2811
00000100:00100000:1.0:1766100063.678137:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432967104:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100063.678180:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bf7662d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432967104:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (252us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100063.678187:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2811
00000100:00100000:1.0:1766100063.678204:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2812
00000100:00100000:1.0:1766100063.678207:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432966976:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100063.678230:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073b447f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432966976:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (241us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100063.678236:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2812
00000100:00100000:1.0:1766100063.678284:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b602d77c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432966976:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100063.678294:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432967104:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100063.678319:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967040
00000100:00100000:1.0:1766100063.678335:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1756
00000100:00100000:1.0:1766100063.678352:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432967040:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766100063.678441:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100063.678502:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432967040:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (442us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100063.678508:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1756
00000100:00080000:1.0:1766100063.678534:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100063.678538:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100063.678541:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e177295 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432967040:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766100063.678674:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d94121f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432966912:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100063.678709:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100063.678728:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100063.678733:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432967168:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100063.678753:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100063.678778:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432966912
00000100:00100000:0.0:1766100063.678801:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1903
00000100:00100000:0.0:1766100063.678805:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432966912:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100063.678834:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432966912:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (125us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100063.678840:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1903
00000100:00100000:0.0:1766100063.678869:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d94121f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432966912:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100063.678904:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967168
00000100:00100000:0.0:1766100063.678921:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1215
00000100:00100000:0.0:1766100063.678924:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432967168:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100063.678950:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432967168:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (197us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100063.678955:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1215
00010000:00080000:0.0:1766100063.678977:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003dd022ac x1851885432967168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100070 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766100063.678986:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003dd022ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432967168:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766100064.253724:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100065.277713:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100066.301673:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100067.325795:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100068.130873:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100068.130951:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100068.131193:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242176
00000100:00100000:3.0:1766100068.131887:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2813
00000100:00100000:3.0:1766100068.131899:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417242176:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100068.132035:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6e3728 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417242176:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 136us (1084us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100068.132044:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2813
00000100:00100000:0.0:1766100068.132986:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242112
00000100:00100000:0.0:1766100068.134468:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1904
00000100:00100000:0.0:1766100068.134497:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417242112:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100068.136854:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417242112:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 2345us (5984us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100068.136883:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1904
00000100:00100000:3.0:1766100068.137775:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100068.137992:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242240
00000100:00100000:3.0:1766100068.138046:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1757
00000100:00100000:3.0:1766100068.138056:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417242240:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100068.138165:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100068.138259:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417242240:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 203us (493us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100068.138266:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1757
00010000:00010000:1.0:1766100068.349929:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100068.805419:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057275e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432967232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100068.805557:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100068.807126:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967232
00000100:00100000:1.0:1766100068.807210:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1905
00000100:00100000:1.0:1766100068.807218:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432967232:12345-0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766100068.807461:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100068.807470:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100068.807474:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100068.807489:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4572
00000100:00080000:0.0:1766100068.807492:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100068.807496:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100068.807519:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000c572263a x1851885432967360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100068.807768:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003dc105a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432967488:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100068.807817:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100068.807867:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432967296:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100068.807896:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100068.807916:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100068.807921:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432967424:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100068.807943:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100068.807986:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967488
00000100:00100000:2.0:1766100068.808021:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1216
00000100:00100000:2.0:1766100068.808025:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432967488:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100068.808782:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432967232:12345-0@lo:400:kworker/u8:0.0 Request processed in 1561us (3225us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100068.808793:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1905
00000100:00100000:0.0:1766100068.808897:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432967360:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766100068.808937:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100068.808977:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967360
00000100:00100000:0.0:1766100068.809005:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1758
00000100:00100000:0.0:1766100068.809009:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432967360:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766100068.809133:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057275e9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432967232:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100068.809177:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967296
00000100:00100000:3.0:1766100068.809211:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967424
00000100:00100000:2.0:1766100068.809225:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432967488:12345-0@lo:400:kworker/u8:0.0 Request processed in 1199us (1409us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100068.809229:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2814
00000100:00100000:3.0:1766100068.809232:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432967296:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100068.809233:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1216
00010000:00080000:2.0:1766100068.809269:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000003dc105a x1851885432967488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100075 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766100068.809274:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079ab5e9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432967296:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (1377us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100068.809280:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2814
00000100:00100000:2.0:1766100068.809282:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003dc105a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432967488:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100068.809292:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2815
00000100:00100000:3.0:1766100068.809295:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432967424:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100068.809335:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949bc664 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432967424:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (1392us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100068.809342:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2815
00000100:00100000:2.0:1766100068.809356:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000f47354 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432967424:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100068.809366:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432967296:0@lo:400:kworker/u8:0.0
00010000:02020000:0.0:1766100068.810593:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100068.810667:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432967360:12345-0@lo:8:kworker/u8:0.0 Request processed in 1654us (1727us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100068.810680:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1758
00000100:00080000:1.0:1766100068.810784:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100068.810789:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100068.810795:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432967360:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766100069.374273:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100070.398028:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100071.421833:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100072.448522:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100073.244749:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100073.244819:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100073.244868:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100073.244933:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242432
00000100:00100000:1.0:1766100073.244993:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1759
00000100:00100000:1.0:1766100073.245003:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417242432:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766100073.245119:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100073.245209:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417242432:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 207us (340us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100073.245219:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1759
00000100:00100000:2.0:1766100073.245811:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242304
00000100:00100000:2.0:1766100073.245861:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1906
00000100:00100000:2.0:1766100073.245866:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417242304:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100073.245929:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417242304:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 63us (1196us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100073.245937:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1906
00000100:00100000:3.0:1766100073.246071:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242368
00000100:00100000:3.0:1766100073.246116:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2816
00000100:00100000:3.0:1766100073.246120:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417242368:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100073.246159:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e177295 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417242368:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 38us (1340us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100073.246165:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2816
00010000:00010000:0.0:1766100073.471448:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100073.920575:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002288850c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432967616:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100073.920681:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100073.920756:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100073.920772:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432967552:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100073.920815:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100073.920870:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967552
00000100:00100000:0.0:1766100073.920931:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1907
00000100:00100000:0.0:1766100073.920939:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432967552:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100073.920997:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432967552:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (181us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100073.921005:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1907
00000100:00100000:0.0:1766100073.921068:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432967552:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100073.921131:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967616
00000100:00100000:0.0:1766100073.921162:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2817
00000100:00100000:0.0:1766100073.921166:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b602d77c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432967616:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766100073.921318:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100073.921324:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100073.921328:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100073.921334:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4577
00000100:00080000:2.0:1766100073.921338:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100073.921342:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100073.921371:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b17b5997 x1851885432967680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100073.921432:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432967744:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100073.921465:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100073.921516:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967744
00000100:00100000:3.0:1766100073.921540:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2818
00000100:00100000:3.0:1766100073.921544:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432967744:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100073.921580:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000037cc3dde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432967744:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (114us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100073.921586:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2818
00000100:00100000:3.0:1766100073.921613:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432967744:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100073.921648:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432967680:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100073.921676:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100073.921712:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967680
00000100:00100000:3.0:1766100073.921733:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1760
00000100:00100000:3.0:1766100073.921737:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432967680:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100073.921835:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100073.921868:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432967680:12345-0@lo:8:kworker/u8:3.0 Request processed in 130us (192us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100073.921875:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1760
00000100:00080000:3.0:1766100073.921897:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100073.921901:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100073.921904:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432967680:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100073.924007:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b602d77c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432967616:12345-0@lo:400:kworker/u8:3.0 Request processed in 2826us (3313us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100073.924031:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2817
00000100:00100000:2.0:1766100073.924403:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100073.924420:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432967808:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100073.924525:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100073.924625:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002288850c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432967616:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100073.924628:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967808
00000100:00100000:3.0:1766100073.924692:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1217
00000100:00100000:3.0:1766100073.924700:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432967808:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100073.924772:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432967808:12345-0@lo:400:kworker/u8:3.0 Request processed in 72us (248us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100073.924778:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1217
00010000:00080000:0.0:1766100073.926456:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000071bbd8cc x1851885432967808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100080 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100073.926500:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432967808:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100074.494590:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100075.517829:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100076.545886:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100077.565937:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100078.368126:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100078.368530:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242560
00000100:00100000:2.0:1766100078.368608:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2819
00000100:00100000:2.0:1766100078.368616:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417242560:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100078.368741:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003dd022ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417242560:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 124us (627us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100078.368750:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2819
00000100:00100000:1.0:1766100078.369311:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100078.370409:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100078.370530:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242496
00000100:00100000:3.0:1766100078.370591:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1908
00000100:00100000:3.0:1766100078.370598:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417242496:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100078.370676:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417242496:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 78us (1367us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100078.370685:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1908
00000100:00100000:2.0:1766100078.373211:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242624
00000100:00100000:2.0:1766100078.373297:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1761
00000100:00100000:2.0:1766100078.373307:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417242624:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:2.0:1766100078.373435:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100078.373559:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417242624:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 252us (3242us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100078.373567:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1761
00010000:00010000:0.0:1766100078.589834:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100079.038494:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100079.038504:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100079.038508:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100079.038518:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4582
00000100:00080000:0.0:1766100079.038522:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100079.038531:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100079.038556:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f5012ca9 x1851885432968000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100079.038893:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432968128:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.038987:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100079.039041:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766100079.039047:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9759d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432967872:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.039082:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100079.039115:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000116a4423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432967936:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.039145:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100079.039165:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100079.039170:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432968064:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.039190:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100079.039240:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967936
00000100:00100000:1.0:1766100079.039296:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968064
00000100:00100000:1.0:1766100079.039322:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2820
00000100:00100000:1.0:1766100079.039329:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432967936:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.039394:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046cfd9d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432967936:12345-0@lo:400:kworker/u8:0.0 Request processed in 64us (248us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100079.039404:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2820
00000100:00100000:1.0:1766100079.039425:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2821
00000100:00100000:1.0:1766100079.039428:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a951f581 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432968064:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.039454:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a951f581 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432968064:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (263us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100079.039460:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2821
00000100:00100000:1.0:1766100079.039522:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432967872
00000100:00100000:1.0:1766100079.039548:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1909
00000100:00100000:1.0:1766100079.039553:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432967872:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.039587:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432967872:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (505us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100079.039594:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1909
00000100:00100000:1.0:1766100079.039660:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432968064:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100079.039849:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5012ca9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968000:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766100079.039887:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100079.039933:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968128
00000100:00100000:3.0:1766100079.039963:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968000
00000100:00100000:3.0:1766100079.039981:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1218
00000100:00100000:3.0:1766100079.039985:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432968128:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100079.040018:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432968128:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (1033us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100079.040028:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1218
00000100:00100000:3.0:1766100079.040042:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1762
00000100:00100000:3.0:1766100079.040045:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432968000:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766100079.040139:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100079.040173:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432968000:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (286us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100079.040181:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1762
00000100:00080000:3.0:1766100079.040214:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100079.040218:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100079.040222:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5012ca9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968000:0@lo:8:kworker/u8:0.0
00000100:00100000:3.0:1766100079.040299:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9759d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432967872:0@lo:400:kworker/u8:0.0
00010000:00080000:3.0:1766100079.040311:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000411dc2e4 x1851885432968128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100086 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:3.0:1766100079.040321:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432968128:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100079.043102:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000116a4423 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432967936:0@lo:400:kworker/u8:0.0
00010000:00010000:0.0:1766100079.613777:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100080.639528:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100081.661922:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100082.685739:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100083.490896:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100083.490958:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100083.491011:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100083.491170:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242816
00000100:00100000:2.0:1766100083.491249:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1763
00000100:00100000:2.0:1766100083.491258:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2e3807e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417242816:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100083.491379:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100083.491510:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2e3807e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417242816:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 250us (624us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100083.491519:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1763
00000100:00100000:3.0:1766100083.491883:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242752
00000100:00100000:3.0:1766100083.491934:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2822
00000100:00100000:3.0:1766100083.491938:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000f47354 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417242752:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100083.491989:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000f47354 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417242752:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 50us (1031us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100083.491995:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2822
00000100:00100000:3.0:1766100083.492019:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242688
00000100:00100000:3.0:1766100083.492119:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1910
00000100:00100000:3.0:1766100083.492124:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417242688:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100083.492148:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417242688:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 25us (1139us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100083.492154:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1910
00010000:00010000:1.0:1766100083.710790:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100084.160026:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100084.160038:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100084.160043:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100084.160053:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4587
00000100:00080000:1.0:1766100084.160058:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100084.160066:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100084.160097:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ecf17fc2 x1851885432968320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100084.160392:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432968192:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100084.160521:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100084.160558:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766100084.160564:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432968448:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100084.160593:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100084.160626:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432968256:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100084.160659:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100084.160678:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100084.160683:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432968384:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100084.160703:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100084.161857:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968448
00000100:00100000:0.0:1766100084.161930:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1219
00000100:00100000:0.0:1766100084.161937:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432968448:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100084.162038:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432968448:12345-0@lo:400:kworker/u8:0.0 Request processed in 101us (1445us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100084.162048:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1219
00010000:00080000:0.0:1766100084.162355:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000065db97cd x1851885432968448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100091 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766100084.162375:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065db97cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432968448:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100084.162516:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968320:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100084.162565:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100084.163181:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968192
00000100:00100000:3.0:1766100084.163227:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1911
00000100:00100000:3.0:1766100084.163231:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432968192:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100084.163281:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432968192:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (2761us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100084.163287:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1911
00000100:00100000:2.0:1766100084.163451:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968256
00000100:00100000:2.0:1766100084.163510:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968384
00000100:00100000:2.0:1766100084.163532:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2823
00000100:00100000:2.0:1766100084.163535:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045cb1471 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432968256:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100084.163579:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045cb1471 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432968256:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (2919us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100084.163585:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2823
00000100:00100000:2.0:1766100084.163598:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2824
00000100:00100000:2.0:1766100084.163601:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068c3be10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432968384:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100084.163628:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068c3be10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432968384:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (2925us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100084.163635:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2824
00000100:00100000:1.0:1766100084.163912:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ffb8f463 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432968384:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100084.163929:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9adb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432968256:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100084.163975:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968320
00000100:00100000:1.0:1766100084.167169:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1764
00000100:00100000:1.0:1766100084.167183:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432968320:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766100084.167298:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100084.167409:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432968320:12345-0@lo:8:kworker/u8:0.0 Request processed in 226us (4844us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100084.167418:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1764
00000100:00100000:0.0:1766100084.167669:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432968192:0@lo:400:kworker/u8:0.0
00000100:00080000:1.0:1766100084.170234:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100084.170252:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100084.170263:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968320:0@lo:8:kworker/u8:0.0
00010000:00010000:0.0:1766100084.735266:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100085.757825:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100086.783276:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100087.806154:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100088.612899:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100088.612966:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100088.613095:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100088.613229:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242880
00000100:00100000:2.0:1766100088.613307:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1912
00000100:00100000:2.0:1766100088.613317:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417242880:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100088.613421:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417242880:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 104us (535us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100088.613430:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1912
00000100:00100000:3.0:1766100088.614009:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243008
00000100:00100000:3.0:1766100088.614052:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1765
00000100:00100000:3.0:1766100088.614058:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417243008:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100088.614167:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100088.614226:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417243008:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 167us (1131us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100088.614234:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1765
00000100:00100000:1.0:1766100088.614361:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417242944
00000100:00100000:1.0:1766100088.614410:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2825
00000100:00100000:1.0:1766100088.614414:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003dc105a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417242944:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100088.614455:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003dc105a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417242944:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 40us (1490us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100088.614462:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2825
00010000:00010000:0.0:1766100088.829832:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100089.280050:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100089.280061:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100089.280066:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100089.280078:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4592
00000100:00080000:1.0:1766100089.280083:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:0.0:1766100089.280084:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432968512:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100089.280092:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100089.280124:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000da5bbb9 x1851885432968640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100089.280189:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432968768:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100089.280223:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100089.280619:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a478d4ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432968704:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100089.280746:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968512
00000100:00100000:0.0:1766100089.280804:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1913
00000100:00100000:0.0:1766100089.280812:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432968512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100089.281809:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100089.281884:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432968512:12345-0@lo:400:kworker/u8:3.0 Request processed in 1070us (1661us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100089.281889:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432968576:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100089.281898:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100089.281902:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1913
00000100:00100000:1.0:1766100089.281933:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100089.281970:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968640:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100089.281982:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968704
00000100:00100000:0.0:1766100089.281983:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432968512:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100089.281998:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100089.282028:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968576
00000100:00100000:0.0:1766100089.282032:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2826
00000100:00100000:0.0:1766100089.282037:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007abdfee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432968704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100089.282041:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968768
00000100:00100000:2.0:1766100089.282055:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2827
00000100:00100000:2.0:1766100089.282060:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079a59718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432968576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100089.282077:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968640
00000100:00100000:0.0:1766100089.282093:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007abdfee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432968704:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (197us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100089.282095:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079a59718 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432968576:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (164us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100089.282101:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1220
00000100:00100000:0.0:1766100089.282102:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2826
00000100:00100000:1.0:1766100089.282109:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432968768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100089.282131:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1766
00000100:00100000:0.0:1766100089.282137:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432968640:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100089.282147:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432968768:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (341us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100089.282182:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1220
00010000:00080000:1.0:1766100089.282227:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000010ae602d x1851885432968768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100096 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100089.282237:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432968768:0@lo:400:kworker/u8:3.0
00010000:02020000:0.0:1766100089.282244:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100089.282276:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432968576:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100089.282291:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432968640:12345-0@lo:8:kworker/u8:3.0 Request processed in 152us (293us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100089.282300:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1766
00000100:00080000:1.0:1766100089.282303:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100089.282309:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100089.282313:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968640:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100089.282444:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a478d4ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432968704:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100089.282451:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2827
00010000:00010000:1.0:1766100089.853752:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100090.877723:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100091.926403:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100092.928442:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100093.724762:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100093.724812:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100093.725068:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243136
00000100:00100000:3.0:1766100093.725140:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2828
00000100:00100000:3.0:1766100093.725148:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061acc5af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417243136:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100093.725463:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243072
00000100:00100000:2.0:1766100093.725525:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1914
00000100:00100000:2.0:1766100093.725529:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417243072:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100093.725988:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417243072:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 457us (1233us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100093.725994:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061acc5af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417243136:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 847us (1183us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100093.725997:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1914
00000100:00100000:3.0:1766100093.726000:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2828
00000100:00100000:0.0:1766100093.726198:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100093.726547:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243200
00000100:00100000:1.0:1766100093.726623:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1767
00000100:00100000:1.0:1766100093.726631:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417243200:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766100093.726741:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100093.730804:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417243200:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 4157us (4591us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100093.730828:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1767
00010000:00010000:0.0:1766100093.950795:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100094.397793:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100094.397804:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100094.397810:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100094.397821:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4597
00000100:00080000:1.0:1766100094.397826:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100094.397834:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100094.397865:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000010ae602d x1851885432968960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100094.397931:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed297b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969024:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398016:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100094.398059:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100094.398064:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432968896:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398083:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100094.398105:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968960:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398142:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100094.398182:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969024
00000100:00100000:1.0:1766100094.398232:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968896
00000100:00100000:1.0:1766100094.398253:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2829
00000100:00100000:1.0:1766100094.398260:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8700567 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398325:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8700567 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969024:12345-0@lo:400:kworker/u8:3.0 Request processed in 65us (310us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100094.398334:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2829
00000100:00100000:1.0:1766100094.398355:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2830
00000100:00100000:1.0:1766100094.398359:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7ebad2d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432968896:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398384:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7ebad2d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432968896:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (301us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100094.398390:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2830
00000100:00100000:1.0:1766100094.398434:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432968896:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398443:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed297b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969024:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.398467:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968960
00000100:00100000:1.0:1766100094.398515:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1768
00000100:00100000:1.0:1766100094.398519:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aea833b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432968960:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100094.399916:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432969088:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100094.400122:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432968832:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100094.400170:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100094.400234:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432968832
00000100:00100000:0.0:1766100094.400285:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1915
00000100:00100000:0.0:1766100094.400289:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432968832:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100094.400333:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432968832:12345-0@lo:400:kworker/u8:3.0 Request processed in 44us (164us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100094.400340:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1915
00000100:00100000:0.0:1766100094.400393:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432968832:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100094.401043:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:02020000:1.0:1766100094.401160:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100094.401223:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aea833b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432968960:12345-0@lo:8:kworker/u8:3.0 Request processed in 2703us (3081us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100094.401232:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1768
00000100:00100000:1.0:1766100094.401253:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969088
00000100:00100000:1.0:1766100094.401285:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1221
00000100:00100000:1.0:1766100094.401289:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b44b2785 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432969088:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100094.401325:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b44b2785 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432969088:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (1338us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100094.401330:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1221
00000100:00080000:2.0:1766100094.401466:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100094.401472:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100094.401505:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432968960:0@lo:8:kworker/u8:3.0
00010000:00080000:2.0:1766100094.401540:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000076bb4d8b x1851885432969088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100101 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100094.401551:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432969088:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100094.973790:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100095.998344:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100097.022438:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100098.049214:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100098.846350:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100098.846423:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100098.846444:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243328
00000100:00100000:0.0:1766100098.846470:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100098.846505:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243264
00000100:00100000:1.0:1766100098.846542:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1916
00000100:00100000:2.0:1766100098.846542:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2831
00000100:00100000:1.0:1766100098.846550:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417243264:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100098.846550:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043db2b7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417243328:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100098.846632:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417243264:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 82us (209us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100098.846632:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043db2b7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417243328:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 82us (295us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100098.846640:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1916
00000100:00100000:2.0:1766100098.846640:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2831
00000100:00100000:3.0:1766100098.853629:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243392
00000100:00100000:3.0:1766100098.853731:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1769
00000100:00100000:3.0:1766100098.853741:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003520602a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417243392:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100098.853864:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100098.853981:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003520602a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417243392:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 240us (7511us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100098.853991:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1769
00010000:00010000:1.0:1766100099.071373:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100099.518945:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0c228e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432969152:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100099.519045:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100099.519117:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969152
00000100:00100000:1.0:1766100099.519178:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1917
00000100:00100000:1.0:1766100099.519185:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432969152:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100099.519245:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432969152:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (202us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100099.519258:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1917
00000100:00100000:1.0:1766100099.519313:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0c228e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432969152:0@lo:400:kworker/u8:0.0
00000100:00080000:2.0:1766100099.521554:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100099.521563:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100099.521568:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100099.521576:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4602
00000100:00080000:2.0:1766100099.521581:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100099.521587:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100099.521615:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008834c3ad x1851885432969280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100099.521678:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432969280:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766100099.521767:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100099.521877:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100099.521894:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432969216:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100099.521930:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100099.522091:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969280
00000100:00100000:3.0:1766100099.522297:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1770
00000100:00100000:3.0:1766100099.522305:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432969280:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:3.0:1766100099.522402:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100099.522571:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969216
00000100:00100000:2.0:1766100099.522612:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2832
00000100:00100000:2.0:1766100099.522616:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed297b67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432969216:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.522673:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed297b67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432969216:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (743us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100099.522680:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2832
00000100:00100000:2.0:1766100099.522745:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432969216:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.522825:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009db79f91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432969408:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.522855:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100099.522881:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969344:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.522904:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100099.522927:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969344
00000100:00100000:2.0:1766100099.522949:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2833
00000100:00100000:2.0:1766100099.522953:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbff49d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969344:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.522983:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbff49d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969344:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (79us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100099.522989:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2833
00000100:00100000:2.0:1766100099.523013:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1f26126 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969344:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.523030:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969408
00000100:00100000:2.0:1766100099.523053:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1222
00000100:00100000:2.0:1766100099.523057:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432969408:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100099.523101:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432969408:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (246us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100099.523108:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1222
00000100:00100000:3.0:1766100099.524811:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432969280:12345-0@lo:8:kworker/u8:0.0 Request processed in 2498us (3038us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100099.524827:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1770
00000100:00080000:0.0:1766100099.524952:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100099.524958:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100099.524963:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432969280:0@lo:8:kworker/u8:0.0
00010000:00080000:2.0:1766100099.527015:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009db79f91 x1851885432969408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100106 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766100099.527039:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009db79f91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432969408:0@lo:400:kworker/u8:0.0
00010000:00010000:2.0:1766100100.098294:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100101.118293:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100102.141787:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100103.166197:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100103.970308:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100103.970371:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100103.970512:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100103.970615:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243456
00000100:00100000:1.0:1766100103.970685:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1918
00000100:00100000:1.0:1766100103.970694:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417243456:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:1.0:1766100103.970785:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417243456:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 92us (414us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100103.970793:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1918
00000100:00100000:0.0:1766100103.970975:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243520
00000100:00100000:0.0:1766100103.971022:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2834
00000100:00100000:0.0:1766100103.971027:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1f26126 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417243520:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100103.971080:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1f26126 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417243520:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 52us (567us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100103.971087:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2834
00000100:00100000:3.0:1766100103.971226:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243584
00000100:00100000:3.0:1766100103.971260:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1771
00000100:00100000:3.0:1766100103.971265:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417243584:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100103.971366:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100103.971417:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417243584:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 151us (1123us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100103.971425:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1771
00010000:00010000:0.0:1766100104.189868:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100104.641470:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100104.641495:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100104.641499:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100104.641509:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4607
00000100:00080000:1.0:1766100104.641513:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100104.641520:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100104.641545:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000362df469 x1851885432969600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100104.642089:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432969600:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766100104.642181:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100104.642896:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969600
00000100:00100000:1.0:1766100104.642957:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1772
00000100:00100000:1.0:1766100104.642964:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432969600:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100104.643068:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432969728:0@lo:400:kworker/u8:0.0
00010000:02020000:1.0:1766100104.643075:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100104.643118:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100104.643134:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432969600:12345-0@lo:8:kworker/u8:0.0 Request processed in 170us (955us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100104.643140:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766100104.643142:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1772
00000100:00100000:2.0:1766100104.643146:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432969472:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100104.643159:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969728
00000100:00100000:2.0:1766100104.643175:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100104.643187:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1223
00000100:00100000:1.0:1766100104.643191:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432969728:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100104.643201:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969664:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100104.643228:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100104.643233:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432969728:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (116us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100104.643239:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1223
00000100:00100000:2.0:1766100104.643245:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100104.643250:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bfc1948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432969536:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100104.643264:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969664
00000100:00100000:2.0:1766100104.643271:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100104.643313:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969472
00000100:00100000:2.0:1766100104.643340:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1919
00000100:00100000:2.0:1766100104.643343:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432969472:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100104.643373:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432969472:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (197us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100104.643379:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1919
00000100:00100000:0.0:1766100104.644095:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432969472:0@lo:400:kworker/u8:0.0
00010000:00080000:0.0:1766100104.644112:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000010ae602d x1851885432969728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100111 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766100104.644124:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432969728:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766100104.644164:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100104.644169:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100104.644174:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432969600:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100104.645731:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969536
00000100:00100000:1.0:1766100104.645797:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2835
00000100:00100000:1.0:1766100104.645805:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969664:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100104.645919:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fb915dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969664:12345-0@lo:400:kworker/u8:0.0 Request processed in 114us (2690us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100104.645928:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2835
00000100:00100000:1.0:1766100104.645947:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2836
00000100:00100000:1.0:1766100104.645950:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002684e04b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432969536:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100104.645978:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002684e04b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432969536:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (2706us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100104.645984:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2836
00000100:00100000:1.0:1766100104.646033:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bfc1948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432969536:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100104.646061:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969664:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766100105.214422:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100106.250433:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100107.263261:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100108.285918:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100109.087097:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100109.087151:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100109.087207:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100109.087291:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243776
00000100:00100000:2.0:1766100109.087336:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243712
00000100:00100000:3.0:1766100109.087361:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1773
00000100:00100000:3.0:1766100109.087371:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417243776:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100109.087512:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100109.087518:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2837
00000100:00100000:0.0:1766100109.087525:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066062a18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417243712:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100109.087596:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417243776:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 224us (387us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100109.087605:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1773
00000100:00100000:1.0:1766100109.087658:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243648
00000100:00100000:1.0:1766100109.087699:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1920
00000100:00100000:1.0:1766100109.087704:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417243648:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100109.088067:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066062a18 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417243712:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 541us (916us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100109.088075:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2837
00000100:00100000:1.0:1766100109.088124:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417243648:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 419us (1039us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100109.088131:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1920
00010000:00010000:2.0:1766100109.311354:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100109.764441:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100109.764450:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100109.764455:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766100109.764457:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766100109.764466:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4613
00000100:00080000:3.0:1766100109.764471:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100109.764518:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100109.764550:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000d779487b x1851885432969920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100109.764639:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000068a35948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432969856:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100109.764693:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432969792:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100109.764741:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100109.765278:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969984:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100109.765444:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d779487b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432969920:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100109.765795:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432970048:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100109.765828:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100109.765892:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969792
00000100:00100000:1.0:1766100109.765940:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100109.765947:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1921
00000100:00100000:2.0:1766100109.765954:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432969792:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100109.765976:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970048
00000100:00100000:1.0:1766100109.766024:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969920
00000100:00100000:2.0:1766100109.766028:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432969792:12345-0@lo:400:kworker/u8:3.0 Request processed in 74us (1288us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100109.766037:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1921
00000100:00100000:0.0:1766100109.766043:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1224
00000100:00100000:1.0:1766100109.766050:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1774
00000100:00100000:0.0:1766100109.766053:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432970048:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100109.766059:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432969920:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100109.766095:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5be2c2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432969792:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100109.766106:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432970048:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (278us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100109.766119:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1224
00010000:00080000:2.0:1766100109.766126:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000bf96017c x1851885432970048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100116 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100109.766138:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf96017c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432970048:0@lo:400:kworker/u8:3.0
00010000:02020000:1.0:1766100109.766171:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100109.766220:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432969920:12345-0@lo:8:kworker/u8:3.0 Request processed in 163us (281us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100109.766227:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1774
00000100:00100000:3.0:1766100109.766374:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100109.766498:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766100109.766518:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100109.766524:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100109.766528:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d779487b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432969920:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100109.766540:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969856
00000100:00100000:1.0:1766100109.766578:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432969984
00000100:00100000:1.0:1766100109.766599:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2838
00000100:00100000:1.0:1766100109.766603:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8d6226a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432969856:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100109.766643:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8d6226a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432969856:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (270us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100109.766651:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2838
00000100:00100000:1.0:1766100109.766663:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2839
00000100:00100000:1.0:1766100109.766667:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969984:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100109.766692:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bfc1948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432969984:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (196us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100109.766698:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2839
00000100:00100000:1.0:1766100109.766725:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b5b6926 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432969984:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100109.766746:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000068a35948 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432969856:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100110.338564:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100111.357737:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100112.388805:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100113.407529:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100114.204636:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100114.204699:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100114.204725:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243840
00000100:00100000:2.0:1766100114.204801:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1922
00000100:00100000:2.0:1766100114.204810:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417243840:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100114.204911:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417243840:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 102us (289us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100114.204921:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1922
00000100:00100000:3.0:1766100114.205060:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243904
00000100:00100000:3.0:1766100114.205103:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2840
00000100:00100000:3.0:1766100114.205107:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417243904:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100114.205143:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca6dd622 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417243904:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 35us (445us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100114.205149:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2840
00000100:00100000:1.0:1766100114.207952:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100114.208077:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417243968
00000100:00100000:0.0:1766100114.208137:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1775
00000100:00100000:0.0:1766100114.208145:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000da5bbb9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417243968:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:0.0:1766100114.208248:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100114.208418:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000da5bbb9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417243968:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 273us (477us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100114.208426:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1775
00010000:00010000:1.0:1766100114.429739:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100114.877987:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100114.877995:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100114.878000:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100114.878011:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4618
00000100:00080000:2.0:1766100114.878016:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100114.878024:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100114.878032:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100114.878051:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008834c3ad x1851885432970240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100114.878064:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432970112:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.878115:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432970240:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100114.878178:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432970176:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100114.878217:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100114.878225:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432970368:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100114.878270:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100114.878299:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100114.878344:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970112
00000100:00100000:2.0:1766100114.878381:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970240
00000100:00100000:0.0:1766100114.878406:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1923
00000100:00100000:0.0:1766100114.878414:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432970112:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.878424:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1776
00000100:00100000:2.0:1766100114.878431:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002288850c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432970240:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100114.878551:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100114.878600:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002288850c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432970240:12345-0@lo:8:kworker/u8:3.0 Request processed in 168us (304us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100114.878607:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1776
00000100:00080000:2.0:1766100114.878651:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100114.878655:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100114.878660:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008834c3ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432970240:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100114.878710:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0c228e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432970304:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.878739:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100114.878773:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970304
00000100:00100000:2.0:1766100114.878801:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2841
00000100:00100000:2.0:1766100114.878805:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008834c3ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432970304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.878841:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008834c3ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432970304:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (103us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100114.878846:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2841
00000100:00100000:0.0:1766100114.878926:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432970112:12345-0@lo:400:kworker/u8:3.0 Request processed in 508us (656us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100114.878938:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1923
00000100:00100000:0.0:1766100114.879038:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432970112:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.879191:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0c228e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432970304:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100114.879305:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100114.879408:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100114.879499:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970368
00000100:00100000:1.0:1766100114.879538:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1225
00000100:00100000:1.0:1766100114.879543:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432970368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100114.879580:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432970368:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (173us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100114.879586:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1225
00010000:00080000:3.0:1766100114.879653:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000071bbd8cc x1851885432970368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100121 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100114.879663:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432970368:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.880598:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970176
00000100:00100000:2.0:1766100114.882297:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2842
00000100:00100000:2.0:1766100114.882312:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a478d4ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432970176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100114.883080:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a478d4ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432970176:12345-0@lo:400:kworker/u8:3.0 Request processed in 766us (3772us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100114.883091:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2842
00000100:00100000:1.0:1766100114.883199:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432970176:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100115.454723:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100116.484978:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100117.502215:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100118.530196:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100119.325152:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100119.325719:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100119.325857:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244096
00000100:00100000:0.0:1766100119.325930:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244032
00000100:00100000:2.0:1766100119.325949:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2843
00000100:00100000:2.0:1766100119.325960:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417244096:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100119.326592:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1924
00000100:00100000:1.0:1766100119.326600:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417244032:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100119.326622:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068a35948 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417244096:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 660us (1481us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100119.326632:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2843
00000100:00100000:1.0:1766100119.326638:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417244032:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 36us (920us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100119.326646:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1924
00000100:00100000:2.0:1766100119.331130:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100119.334342:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244160
00000100:00100000:3.0:1766100119.334435:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1777
00000100:00100000:3.0:1766100119.334444:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417244160:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100119.334603:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100119.334718:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417244160:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 273us (3597us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100119.334727:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1777
00010000:00010000:0.0:1766100119.550564:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100119.998162:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100119.998172:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100119.998176:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100119.998186:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4623
00000100:00080000:2.0:1766100119.998190:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100119.998196:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100119.998222:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a9408511 x1851885432970560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100119.998292:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432970432:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100119.998382:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100119.998415:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100119.998421:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432970688:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100119.998448:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100119.998494:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432970624:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100119.998523:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100119.998528:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432970560:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100119.998541:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100119.998545:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432970496:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100119.998559:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100119.998564:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100119.998594:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970560
00000100:00100000:2.0:1766100119.998595:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970688
00000100:00100000:3.0:1766100119.998630:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1226
00000100:00100000:3.0:1766100119.998636:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432970688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100119.998691:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432970688:12345-0@lo:400:kworker/u8:3.0 Request processed in 54us (242us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100119.998699:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1226
00000100:00100000:3.0:1766100119.998718:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1778
00000100:00100000:3.0:1766100119.998721:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432970560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100119.998835:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100119.998868:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432970560:12345-0@lo:8:kworker/u8:3.0 Request processed in 145us (309us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100119.998874:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1778
00000100:00100000:1.0:1766100119.999209:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970624
00000100:00100000:1.0:1766100119.999243:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970496
00000100:00100000:1.0:1766100119.999267:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2844
00000100:00100000:1.0:1766100119.999271:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432970624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100119.999316:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dcf37af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432970624:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (793us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100119.999322:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2844
00000100:00100000:1.0:1766100119.999335:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2845
00000100:00100000:1.0:1766100119.999339:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432970496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100119.999365:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004004294c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432970496:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (800us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100119.999371:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2845
00010000:00080000:1.0:1766100119.999423:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000071bbd8cc x1851885432970688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100126 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100119.999443:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432970688:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100119.999519:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432970496:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100119.999531:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001629b413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432970624:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100119.999569:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100119.999574:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100119.999578:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432970560:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100120.000043:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970432
00000100:00100000:1.0:1766100120.000074:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1925
00000100:00100000:1.0:1766100120.000078:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432970432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100120.000127:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432970432:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (1748us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100120.000133:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1925
00000100:00100000:1.0:1766100120.000188:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432970432:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100120.574053:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100121.597945:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100122.621831:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100123.646878:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100124.457757:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100124.462789:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244352
00000100:00100000:3.0:1766100124.464266:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1779
00000100:00100000:3.0:1766100124.464277:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417244352:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:3.0:1766100124.464390:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100124.464501:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417244352:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 224us (6757us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100124.464510:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1779
00000100:00100000:0.0:1766100124.468564:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100124.468613:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100124.468719:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244224
00000100:00100000:0.0:1766100124.468768:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1926
00000100:00100000:0.0:1766100124.468775:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417244224:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100124.468851:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417244224:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 75us (237us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100124.468858:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1926
00000100:00100000:0.0:1766100124.468948:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244288
00000100:00100000:0.0:1766100124.468978:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2846
00000100:00100000:0.0:1766100124.468982:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417244288:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:0.0:1766100124.469015:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e46356a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417244288:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 33us (463us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100124.469022:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2846
00010000:00010000:2.0:1766100124.669881:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100125.118408:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100125.118417:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100125.118422:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100125.118432:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4628
00000100:00080000:1.0:1766100125.118436:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100125.118442:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100125.118450:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432970816:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.118467:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000010ae602d x1851885432970880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100125.118530:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432970944:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100125.118563:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100125.118564:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432970880:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100125.118646:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432970752:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100125.118707:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432971008:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.118847:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100125.118870:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970816
00000100:00100000:1.0:1766100125.118886:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100125.118928:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2847
00000100:00100000:1.0:1766100125.118934:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970944
00000100:00100000:2.0:1766100125.118936:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432970816:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.118958:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2848
00000100:00100000:1.0:1766100125.118961:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432970944:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.118988:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001629b413 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432970944:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (102us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100125.118988:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038aa4304 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432970816:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (439us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100125.118996:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2847
00000100:00100000:1.0:1766100125.119004:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2848
00000100:00100000:1.0:1766100125.119037:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000983dda28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432970944:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100125.119037:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432970816:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.119080:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970880
00000100:00100000:1.0:1766100125.119106:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1780
00000100:00100000:1.0:1766100125.119110:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432970880:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100125.119209:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100125.119244:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432970880:12345-0@lo:8:kworker/u8:3.0 Request processed in 133us (398us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100125.119250:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1780
00000100:00080000:1.0:1766100125.119273:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100125.119277:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100125.119280:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432970880:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100125.119346:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100125.119426:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100125.119496:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971008
00000100:00100000:1.0:1766100125.119512:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432970752
00000100:00100000:2.0:1766100125.119534:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1227
00000100:00100000:2.0:1766100125.119539:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432971008:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100125.119588:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432971008:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (163us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100125.119595:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1227
00010000:00080000:0.0:1766100125.119615:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b17b5997 x1851885432971008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100132 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100125.119626:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432971008:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.119864:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1927
00000100:00100000:1.0:1766100125.119869:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432970752:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100125.119923:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432970752:12345-0@lo:400:kworker/u8:3.0 Request processed in 53us (578us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100125.119930:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1927
00000100:00100000:3.0:1766100125.120189:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432970752:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100125.693992:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100126.717842:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100127.742447:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100128.770912:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100129.566461:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100129.566579:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244416
00000100:00100000:1.0:1766100129.566836:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1928
00000100:00100000:1.0:1766100129.566844:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417244416:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:2.0:1766100129.569715:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100129.569930:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417244416:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 3084us (3475us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100129.569944:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1928
00000100:00100000:3.0:1766100129.570902:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244480
00000100:00100000:3.0:1766100129.570952:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2849
00000100:00100000:3.0:1766100129.570959:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417244480:12345-192.168.203.52@tcp:400:kworker/u8:1.0
00000100:00100000:3.0:1766100129.571025:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aef029b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417244480:12345-192.168.203.52@tcp:400:kworker/u8:1.0 Request processed in 66us (1317us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100129.571031:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2849
00000100:00100000:2.0:1766100129.571581:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100129.571712:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244544
00000100:00100000:1.0:1766100129.571742:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1781
00000100:00100000:1.0:1766100129.571746:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417244544:12345-192.168.203.52@tcp:8:kworker/u8:1.0
00010000:02020000:1.0:1766100129.571838:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100129.571924:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417244544:12345-192.168.203.52@tcp:8:kworker/u8:1.0 Request processed in 175us (344us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100129.571931:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1781
00010000:00010000:1.0:1766100129.790818:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100130.237739:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100130.237747:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100130.237750:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1766100130.237757:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432971136:0@lo:400:kworker/u8:0.0
00000100:00080000:0.0:1766100130.237765:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4633
00000100:00080000:0.0:1766100130.237767:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100130.237773:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100130.237792:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004e9759d0 x1851885432971200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100130.237823:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100130.237843:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432971328:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100130.237851:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100130.237867:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5012ca9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432971264:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100130.237884:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100130.237884:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100130.237952:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971136
00000100:00100000:2.0:1766100130.237992:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971264
00000100:00100000:2.0:1766100130.238011:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2850
00000100:00100000:2.0:1766100130.238016:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432971136:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100130.238025:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2851
00000100:00100000:3.0:1766100130.238029:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432971264:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100130.238061:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069da214f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432971136:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (239us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100130.238070:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2850
00000100:00100000:0.0:1766100130.238133:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100130.238140:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432971072:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100130.238167:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100130.238190:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971328
00000100:00100000:0.0:1766100130.238211:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1228
00000100:00100000:0.0:1766100130.238214:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432971328:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100130.238245:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432971328:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (362us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100130.238251:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1228
00010000:00080000:0.0:1766100130.238279:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ecf17fc2 x1851885432971328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100137 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:0.0:1766100130.238289:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ecf17fc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432971328:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100130.238372:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9759d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432971200:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100130.238405:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100130.238442:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971200
00000100:00100000:1.0:1766100130.238462:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1782
00000100:00100000:1.0:1766100130.238465:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a03d8323 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432971200:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:1.0:1766100130.238577:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100130.239105:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971072
00000100:00100000:0.0:1766100130.239134:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1929
00000100:00100000:0.0:1766100130.239139:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432971072:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100130.241354:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a65af3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432971264:12345-0@lo:400:kworker/u8:0.0 Request processed in 3316us (3461us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100130.241370:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2851
00000100:00080000:3.0:1766100130.243193:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100130.243201:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100130.243207:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9759d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432971200:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100130.243633:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5012ca9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432971264:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100130.243647:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000411dc2e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432971136:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100130.243662:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432971072:12345-0@lo:400:kworker/u8:0.0 Request processed in 4521us (5493us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100130.243673:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1929
00000100:00100000:3.0:1766100130.245069:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c572263a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432971072:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100130.245956:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a03d8323 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432971200:12345-0@lo:8:kworker/u8:0.0 Request processed in 7485us (7547us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100130.245970:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1782
00010000:00010000:2.0:1766100130.817265:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100131.837690:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100132.868948:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100133.887795:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100134.685228:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100134.685290:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100134.685333:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100134.685744:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244672
00000100:00100000:3.0:1766100134.685923:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2852
00000100:00100000:3.0:1766100134.685931:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417244672:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100134.686090:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244608
00000100:00100000:1.0:1766100134.686127:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1930
00000100:00100000:1.0:1766100134.686132:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417244608:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100134.686204:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417244608:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 71us (914us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100134.686213:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1930
00000100:00100000:2.0:1766100134.686350:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244736
00000100:00100000:2.0:1766100134.686379:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1783
00000100:00100000:2.0:1766100134.686383:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ecb9adb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417244736:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100134.686499:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100134.686536:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ecb9adb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417244736:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 152us (1204us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100134.686543:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1783
00000100:00100000:3.0:1766100134.686802:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecf17fc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417244672:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 870us (1588us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100134.686810:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2852
00010000:00010000:1.0:1766100134.910586:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100135.358663:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100135.358675:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100135.358680:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100135.358691:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4638
00000100:00080000:1.0:1766100135.358695:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100135.358703:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100135.358730:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b3eafafb x1851885432971520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100135.358789:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432971584:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100135.358885:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100135.358940:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100135.358946:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432971456:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100135.358975:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100135.359017:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971584
00000100:00100000:1.0:1766100135.359082:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971456
00000100:00100000:1.0:1766100135.359103:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2853
00000100:00100000:1.0:1766100135.359110:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000983dda28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432971584:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100135.359167:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000983dda28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432971584:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (285us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100135.359175:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2853
00000100:00100000:1.0:1766100135.359193:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2854
00000100:00100000:1.0:1766100135.359196:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043c8fc31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432971456:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100135.359224:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043c8fc31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432971456:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (248us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100135.359229:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2854
00000100:00100000:1.0:1766100135.359272:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432971456:0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100135.359284:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432971584:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100135.359417:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432971392:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100135.359458:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100135.359507:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100135.359513:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432971648:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100135.359539:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100135.359569:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971648
00000100:00100000:2.0:1766100135.359599:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1229
00000100:00100000:2.0:1766100135.359603:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432971648:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100135.359639:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432971648:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (99us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100135.359646:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1229
00010000:00080000:2.0:1766100135.359674:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000362df469 x1851885432971648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100142 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766100135.359685:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432971648:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100135.359728:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432971520:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100135.359756:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100135.359783:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971520
00000100:00100000:2.0:1766100135.359801:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1784
00000100:00100000:2.0:1766100135.359805:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432971520:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:2.0:1766100135.359925:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100135.359964:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432971520:12345-0@lo:8:kworker/u8:0.0 Request processed in 159us (209us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100135.359982:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1784
00000100:00080000:2.0:1766100135.360007:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100135.360011:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100135.360015:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432971520:0@lo:8:kworker/u8:0.0
00000100:00100000:1.0:1766100135.362060:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971392
00000100:00100000:1.0:1766100135.362128:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1931
00000100:00100000:1.0:1766100135.362134:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432971392:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100135.362446:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432971392:12345-0@lo:400:kworker/u8:0.0 Request processed in 311us (2987us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100135.362453:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1931
00000100:00100000:0.0:1766100135.362686:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432971392:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766100135.933769:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100136.958564:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100137.983510:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100139.006703:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100139.805286:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100139.805347:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100139.805406:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244800
00000100:00100000:3.0:1766100139.805522:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1932
00000100:00100000:3.0:1766100139.805531:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417244800:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100139.805636:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417244800:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 105us (289us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100139.805646:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1932
00000100:00100000:2.0:1766100139.805776:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244864
00000100:00100000:1.0:1766100139.805867:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100139.805912:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244928
00000100:00100000:0.0:1766100139.805950:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1785
00000100:00100000:0.0:1766100139.805955:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b510690 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417244928:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100139.806058:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100139.806101:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b510690 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417244928:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 145us (236us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100139.806108:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1785
00000100:00100000:2.0:1766100139.807303:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2855
00000100:00100000:2.0:1766100139.807313:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b24273d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417244864:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100139.808757:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b24273d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417244864:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1438us (3475us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100139.808772:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2855
00010000:00010000:2.0:1766100140.034815:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100140.477695:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100140.477705:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100140.477710:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100140.477721:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4643
00000100:00080000:1.0:1766100140.477726:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100140.477735:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100140.477745:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100140.477768:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000044763d3d x1851885432971840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100140.477772:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000914f119b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432971776:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100140.478154:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a75d9f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432971712:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100140.478200:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100140.478234:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100140.478241:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100140.478247:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a48ec222 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432971968:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100140.478259:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432971904:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100140.478276:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100140.478281:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100140.478308:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000044763d3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432971840:0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766100140.478328:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971776
00000100:00100000:2.0:1766100140.478333:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100140.478367:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971968
00000100:00100000:0.0:1766100140.478377:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971904
00000100:00100000:2.0:1766100140.478391:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971840
00000100:00100000:0.0:1766100140.478397:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2856
00000100:00100000:0.0:1766100140.478403:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432971776:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100140.478407:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1230
00000100:00100000:2.0:1766100140.478410:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432971968:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100140.478459:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb8f463 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432971776:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (224us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100140.478459:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432971968:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (182us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100140.478467:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1230
00000100:00100000:0.0:1766100140.478468:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2856
00000100:00100000:0.0:1766100140.478513:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2857
00000100:00100000:0.0:1766100140.478517:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432971904:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100140.478528:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1786
00000100:00100000:2.0:1766100140.478532:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432971840:12345-0@lo:8:kworker/u8:0.0
00000100:00100000:0.0:1766100140.478545:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065db97cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432971904:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (264us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100140.478552:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2857
00000100:00100000:0.0:1766100140.478608:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432971904:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100140.478618:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000914f119b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432971776:0@lo:400:kworker/u8:0.0
00010000:02020000:2.0:1766100140.478626:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100140.478674:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432971840:12345-0@lo:8:kworker/u8:0.0 Request processed in 140us (340us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100140.478680:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1786
00000100:00080000:0.0:1766100140.478687:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100140.478691:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100140.478694:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000044763d3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432971840:0@lo:8:kworker/u8:0.0
00010000:00080000:2.0:1766100140.478712:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000a48ec222 x1851885432971968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100147 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766100140.478722:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a48ec222 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432971968:0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100140.478859:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432971712
00000100:00100000:3.0:1766100140.478899:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1933
00000100:00100000:3.0:1766100140.478904:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432971712:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:3.0:1766100140.478953:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432971712:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (755us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100140.478960:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1933
00000100:00100000:2.0:1766100140.479341:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a75d9f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432971712:0@lo:400:kworker/u8:0.0
00010000:00010000:1.0:1766100141.055469:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100142.078349:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100143.101697:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100144.128592:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100144.928836:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100144.928907:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100144.929190:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245056
00000100:00100000:2.0:1766100144.929259:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2858
00000100:00100000:2.0:1766100144.929266:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b77e32c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417245056:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100144.929356:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b77e32c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417245056:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 89us (450us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100144.929363:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2858
00000100:00100000:1.0:1766100144.929854:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100144.929889:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245120
00000100:00100000:1.0:1766100144.929913:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1787
00000100:00100000:1.0:1766100144.929917:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000678c16b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245120:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100144.930014:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100144.930052:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000678c16b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245120:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 132us (199us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100144.930059:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1787
00000100:00100000:3.0:1766100144.930734:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417244992
00000100:00100000:3.0:1766100144.930803:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1934
00000100:00100000:3.0:1766100144.930808:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417244992:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100144.930856:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417244992:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 47us (2032us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100144.930862:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1934
00010000:00010000:1.0:1766100145.149921:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100145.603003:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100145.603013:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100145.603019:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100145.603029:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4648
00000100:00080000:1.0:1766100145.603033:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100145.603040:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100145.603068:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000043119846 x1851885432972160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100145.603130:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b028421 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432972224:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100145.603215:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100145.603257:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100145.603262:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432972096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100145.603280:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100145.603316:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972224
00000100:00100000:1.0:1766100145.603364:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972096
00000100:00100000:1.0:1766100145.603386:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2859
00000100:00100000:1.0:1766100145.603392:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f42693bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432972224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100145.603449:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f42693bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432972224:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (235us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100145.603458:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2859
00000100:00100000:1.0:1766100145.603504:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2860
00000100:00100000:1.0:1766100145.603508:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432972096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100145.603534:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002af2c706 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432972096:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (253us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100145.603539:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2860
00000100:00100000:1.0:1766100145.603587:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432972096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100145.603597:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b028421 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432972224:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100145.603737:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432972288:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100145.603790:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100145.603809:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100145.603815:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432972032:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100145.603846:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100145.604344:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043119846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432972160:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100145.604383:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100145.604436:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972032
00000100:00100000:0.0:1766100145.604473:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1935
00000100:00100000:0.0:1766100145.604508:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432972032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100145.604556:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432972032:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (710us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100145.604563:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1935
00000100:00100000:0.0:1766100145.604613:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432972032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100145.604738:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972288
00000100:00100000:2.0:1766100145.604793:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972160
00000100:00100000:2.0:1766100145.604810:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1231
00000100:00100000:2.0:1766100145.604814:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d53b4be3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432972288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100145.604855:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d53b4be3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432972288:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (1065us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100145.604862:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1231
00000100:00100000:2.0:1766100145.604998:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1788
00000100:00100000:2.0:1766100145.605002:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432972160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100145.605104:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100145.605137:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432972160:12345-0@lo:8:kworker/u8:3.0 Request processed in 133us (753us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100145.605143:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1788
00000100:00080000:0.0:1766100145.605239:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100145.605244:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100145.605248:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043119846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432972160:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766100145.605271:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000000d48768b x1851885432972288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100152 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100145.605282:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432972288:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100146.173758:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100147.198956:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100148.223424:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100149.245756:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100150.044617:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100150.044683:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100150.044701:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245248
00000100:00100000:2.0:1766100150.044771:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2861
00000100:00100000:2.0:1766100150.044778:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007fb05d30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417245248:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.044873:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007fb05d30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417245248:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 95us (269us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100150.044882:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2861
00000100:00100000:2.0:1766100150.045244:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100150.045338:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245312
00000100:00100000:1.0:1766100150.045372:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1789
00000100:00100000:1.0:1766100150.045377:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245312:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100150.045502:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100150.046517:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245312:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1135us (1271us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100150.046530:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1789
00000100:00100000:3.0:1766100150.048433:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245184
00000100:00100000:3.0:1766100150.048540:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1936
00000100:00100000:3.0:1766100150.048549:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417245184:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100150.048664:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417245184:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 114us (3980us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100150.048672:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1936
00010000:00010000:1.0:1766100150.269734:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100150.718888:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100150.718896:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100150.718900:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100150.718909:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4654
00000100:00080000:2.0:1766100150.718913:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100150.718920:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100150.718929:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100150.718945:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432972416:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.719035:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100150.719095:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972416
00000100:00100000:2.0:1766100150.719158:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2862
00000100:00100000:2.0:1766100150.719165:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432972416:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.719235:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d94121f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432972416:12345-0@lo:400:kworker/u8:3.0 Request processed in 70us (202us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100150.719243:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2862
00000100:00100000:2.0:1766100150.719294:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432972416:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.719340:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000055c81506 x1851885432972480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100150.719375:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a55df54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432972544:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.719397:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100150.719418:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972544
00000100:00100000:2.0:1766100150.719440:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2863
00000100:00100000:2.0:1766100150.719444:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432972544:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.719474:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c56d4390 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432972544:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (77us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100150.719508:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2863
00000100:00100000:2.0:1766100150.719534:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a55df54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432972544:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100150.719795:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432972352:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100150.719837:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100150.719871:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100150.719877:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432972608:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100150.719904:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100150.719932:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432972480:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100150.719958:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100150.720002:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972608
00000100:00100000:0.0:1766100150.720029:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972480
00000100:00100000:0.0:1766100150.720049:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1232
00000100:00100000:0.0:1766100150.720053:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000617de322 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432972608:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100150.720087:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000617de322 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432972608:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (183us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100150.720093:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1232
00000100:00100000:0.0:1766100150.720110:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1790
00000100:00100000:0.0:1766100150.720114:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e40221d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432972480:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100150.720224:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100150.720259:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e40221d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432972480:12345-0@lo:8:kworker/u8:3.0 Request processed in 144us (301us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100150.720266:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1790
00010000:00080000:0.0:1766100150.720301:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f7427f30 x1851885432972608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100157 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100150.720312:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7427f30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432972608:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100150.720338:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100150.720343:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100150.720347:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055c81506 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432972480:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100150.720437:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972352
00000100:00100000:2.0:1766100150.720497:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1937
00000100:00100000:2.0:1766100150.720502:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432972352:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100150.720536:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432972352:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (700us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100150.720542:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1937
00000100:00100000:0.0:1766100150.720759:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a16d6f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432972352:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100151.295770:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100152.318731:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100153.342646:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100154.369240:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100155.166666:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100155.166764:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245376
00000100:00100000:2.0:1766100155.166789:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100155.167423:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1938
00000100:00100000:3.0:1766100155.167434:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417245376:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100155.167577:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417245376:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 144us (921us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100155.167587:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1938
00000100:00100000:1.0:1766100155.168037:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245440
00000100:00100000:1.0:1766100155.168089:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2864
00000100:00100000:1.0:1766100155.168094:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ecb9adb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417245440:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100155.168150:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ecb9adb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417245440:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 55us (1362us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100155.168157:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2864
00000100:00100000:2.0:1766100155.171285:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100155.171400:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245504
00000100:00100000:1.0:1766100155.171455:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1791
00000100:00100000:1.0:1766100155.171463:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245504:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100155.173213:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100155.173350:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245504:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1884us (2075us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100155.173371:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1791
00010000:00010000:2.0:1766100155.393505:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100155.845762:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100155.845770:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100155.845775:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100155.845784:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4659
00000100:00080000:2.0:1766100155.845788:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100155.845795:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100155.845822:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e68d47de x1851885432972800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100155.846042:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432972736:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100155.846128:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100155.846158:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100155.846164:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432972864:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100155.846186:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100155.846215:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e68d47de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432972800:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100155.846243:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100155.846282:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972736
00000100:00100000:2.0:1766100155.846326:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972864
00000100:00100000:2.0:1766100155.846346:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2865
00000100:00100000:2.0:1766100155.846353:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d11b33f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432972736:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100155.846411:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d11b33f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432972736:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (284us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100155.846419:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2865
00000100:00100000:2.0:1766100155.846439:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2866
00000100:00100000:2.0:1766100155.846443:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db0fcd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432972864:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100155.846470:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db0fcd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432972864:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (283us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100155.846508:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2866
00000100:00100000:2.0:1766100155.846531:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972800
00000100:00100000:2.0:1766100155.846551:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1792
00000100:00100000:2.0:1766100155.846554:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432972800:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100155.846654:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100155.846702:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432972800:12345-0@lo:8:kworker/u8:3.0 Request processed in 146us (459us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100155.846709:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1792
00000100:00100000:2.0:1766100155.846765:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432972864:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100155.846778:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a55df54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432972672:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100155.846781:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d4e5a5c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432972736:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766100155.846828:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:0.0:1766100155.846829:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766100155.846834:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100155.846839:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e68d47de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432972800:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100155.846864:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972672
00000100:00100000:0.0:1766100155.846899:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1939
00000100:00100000:0.0:1766100155.846903:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432972672:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100155.846932:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432972672:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (105us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100155.846937:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1939
00000100:00100000:0.0:1766100155.846976:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a55df54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432972672:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100155.846985:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100155.846990:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000506576fd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432972928:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100155.847008:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100155.847857:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972928
00000100:00100000:2.0:1766100155.847902:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1233
00000100:00100000:2.0:1766100155.847906:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432972928:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100155.847985:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432972928:12345-0@lo:400:kworker/u8:3.0 Request processed in 72us (970us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100155.847992:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1233
00010000:00080000:0.0:1766100155.849071:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000506576fd x1851885432972928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100162 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100155.849090:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000506576fd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432972928:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100156.414540:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100157.437740:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100158.463032:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100159.485731:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100160.288452:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100160.288549:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100160.288651:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245696
00000100:00100000:1.0:1766100160.288703:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1793
00000100:00100000:1.0:1766100160.288710:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245696:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100160.288807:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100160.288880:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245696:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 169us (435us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100160.288888:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1793
00000100:00100000:2.0:1766100160.289608:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245568
00000100:00100000:2.0:1766100160.289666:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1940
00000100:00100000:2.0:1766100160.289671:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417245568:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100160.289719:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417245568:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 47us (1171us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100160.289725:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1940
00000100:00100000:1.0:1766100160.290252:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100160.291231:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245632
00000100:00100000:2.0:1766100160.291289:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2867
00000100:00100000:2.0:1766100160.291295:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b028421 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417245632:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100160.291655:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b028421 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417245632:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 358us (1403us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100160.291663:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2867
00010000:00010000:1.0:1766100160.509788:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100160.957816:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100160.957823:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100160.957826:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100160.957835:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4664
00000100:00080000:2.0:1766100160.957838:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100160.957844:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100160.957855:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c51db5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432973056:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100160.957868:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000df8193b4 x1851885432973120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100160.957915:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078059cfe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432973184:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.957927:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100160.957994:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432972992:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100160.958880:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432973248:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.959198:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df8193b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432973120:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100160.959230:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100160.959269:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973056
00000100:00100000:3.0:1766100160.959305:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2868
00000100:00100000:3.0:1766100160.959311:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432973056:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.959351:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d779487b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432973056:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (1427us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100160.959359:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2868
00000100:00100000:3.0:1766100160.959392:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c51db5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432973056:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.959423:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973120
00000100:00100000:3.0:1766100160.959438:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1794
00000100:00100000:3.0:1766100160.959441:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432973120:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766100160.959554:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100160.959580:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432973120:12345-0@lo:8:kworker/u8:2.0 Request processed in 138us (350us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100160.959585:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1794
00000100:00080000:3.0:1766100160.959603:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100160.959607:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100160.959610:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df8193b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432973120:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100160.960274:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100160.961194:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973184
00000100:00100000:3.0:1766100160.961243:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2869
00000100:00100000:3.0:1766100160.961248:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006df27fbb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432973184:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.961311:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006df27fbb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432973184:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (1040us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100160.961318:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2869
00000100:00100000:0.0:1766100160.962844:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100160.963122:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078059cfe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432973184:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100160.963212:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100160.963660:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973248
00000100:00100000:2.0:1766100160.963824:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1234
00000100:00100000:2.0:1766100160.963830:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000050b9c6b5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432973248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.963996:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432972992
00000100:00100000:3.0:1766100160.964021:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1941
00000100:00100000:3.0:1766100160.964025:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432972992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100160.964056:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432972992:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (1218us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100160.964063:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1941
00000100:00100000:0.0:1766100160.964193:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432972992:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100160.964349:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000050b9c6b5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432973248:12345-0@lo:400:kworker/u8:2.0 Request processed in 518us (1136us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100160.964355:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1234
00010000:00080000:1.0:1766100160.964398:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ea6f516a x1851885432973248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100167 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766100160.964409:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432973248:0@lo:400:kworker/u8:2.0
00010000:00010000:2.0:1766100161.533718:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100162.558254:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100163.581833:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100164.606435:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100165.409227:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100165.409288:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100165.409338:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100165.409360:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245888
00000100:00100000:3.0:1766100165.409425:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1795
00000100:00100000:3.0:1766100165.409432:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245888:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100165.409567:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100165.409672:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245824
00000100:00100000:3.0:1766100165.409675:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417245888:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 241us (385us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100165.409686:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1795
00000100:00100000:2.0:1766100165.409748:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2870
00000100:00100000:2.0:1766100165.409756:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417245824:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100165.412454:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfd14c2b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417245824:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 2685us (3228us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100165.412473:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2870
00000100:00100000:0.0:1766100165.412971:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245760
00000100:00100000:0.0:1766100165.413032:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1942
00000100:00100000:0.0:1766100165.413039:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417245760:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100165.413135:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417245760:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 96us (3798us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100165.413142:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1942
00010000:00010000:1.0:1766100165.630605:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100166.084958:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:0.0:1766100166.084965:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100166.084974:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100166.084979:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100166.084991:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4669
00000100:00100000:1.0:1766100166.084992:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432973312:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100166.084996:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100166.085006:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100166.085048:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007bef605c x1851885432973440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100166.085079:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100166.085177:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c6056fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432973376:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100166.085233:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007bef605c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432973440:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100166.085246:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100166.085358:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432973568:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100166.086281:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100166.086292:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2e7680e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432973504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100166.086332:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100166.086393:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973312
00000100:00100000:3.0:1766100166.086445:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1943
00000100:00100000:3.0:1766100166.086452:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432973312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100166.086517:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432973312:12345-0@lo:400:kworker/u8:3.0 Request processed in 64us (1441us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100166.086524:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1943
00000100:00100000:3.0:1766100166.086557:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973376
00000100:00100000:3.0:1766100166.086581:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973504
00000100:00100000:3.0:1766100166.086607:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2871
00000100:00100000:3.0:1766100166.086611:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000923c0bd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432973376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100166.086644:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000923c0bd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432973376:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (1402us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100166.086650:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2871
00000100:00100000:3.0:1766100166.086661:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2872
00000100:00100000:3.0:1766100166.086663:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027be1d19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432973504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100166.086681:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027be1d19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432973504:12345-0@lo:400:kworker/u8:3.0 Request processed in 17us (350us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100166.086685:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2872
00000100:00100000:3.0:1766100166.086727:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2e7680e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432973504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100166.086738:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c6056fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432973376:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100166.087074:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100166.087251:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432973312:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100166.087291:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100166.087555:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973568
00000100:00100000:3.0:1766100166.087601:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973440
00000100:00100000:3.0:1766100166.087623:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1235
00000100:00100000:3.0:1766100166.087627:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432973568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100166.087664:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1796
00000100:00100000:0.0:1766100166.087674:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432973440:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100166.087686:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432973568:12345-0@lo:400:kworker/u8:3.0 Request processed in 56us (611us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100166.087695:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1235
00010000:02020000:0.0:1766100166.087776:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100166.087858:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432973440:12345-0@lo:8:kworker/u8:3.0 Request processed in 184us (573us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100166.087867:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1796
00010000:00080000:0.0:1766100166.087922:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000055153256 x1851885432973568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100173 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100166.087936:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432973568:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766100166.088978:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100166.088986:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100166.088991:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007bef605c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432973440:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766100166.653703:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100167.678390:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100168.701961:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100169.726058:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100170.524683:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100170.524735:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100170.524828:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100170.524869:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417245952
00000100:00100000:3.0:1766100170.524948:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1944
00000100:00100000:3.0:1766100170.524955:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417245952:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100170.525036:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417245952:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 80us (300us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100170.525043:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1944
00000100:00100000:0.0:1766100170.525152:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246016
00000100:00100000:0.0:1766100170.525192:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2873
00000100:00100000:0.0:1766100170.525197:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246016:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100170.525232:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005f16d38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246016:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 34us (405us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100170.525239:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2873
00000100:00100000:1.0:1766100170.525437:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246080
00000100:00100000:0.0:1766100170.526150:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1797
00000100:00100000:0.0:1766100170.526159:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417246080:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100170.526280:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100170.526357:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417246080:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 197us (1686us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100170.526365:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1797
00010000:00010000:2.0:1766100170.749773:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100171.199070:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432973632:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199167:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100171.199284:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973632
00000100:00100000:3.0:1766100171.199350:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1945
00000100:00100000:3.0:1766100171.199357:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432973632:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766100171.199387:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100171.199393:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100171.199398:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100171.199403:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4674
00000100:00080000:0.0:1766100171.199406:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100171.199410:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100171.199434:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000042e452f9 x1851885432973760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100171.199616:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432973632:12345-0@lo:400:kworker/u8:2.0 Request processed in 258us (450us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100171.199621:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432973632:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100171.199625:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1945
00000100:00100000:2.0:1766100171.199640:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100171.199647:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432973888:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100171.199663:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432973696:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199672:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100171.199692:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100171.199704:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973888
00000100:00100000:3.0:1766100171.199710:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100171.199714:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009470f4e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432973824:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199733:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1236
00000100:00100000:3.0:1766100171.199735:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100171.199737:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040addeb2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432973888:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100171.199762:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000042e452f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432973760:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199769:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040addeb2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432973888:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (97us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100171.199775:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1236
00000100:00100000:3.0:1766100171.199784:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:2.0:1766100171.199799:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000003c0fbdcb x1851885432973888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100178 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766100171.199809:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432973888:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199839:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973696
00000100:00100000:2.0:1766100171.199864:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973824
00000100:00100000:2.0:1766100171.199880:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2874
00000100:00100000:2.0:1766100171.199884:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7b8f472 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432973696:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199916:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7b8f472 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432973696:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (225us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100171.199925:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2874
00000100:00100000:2.0:1766100171.199935:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2875
00000100:00100000:2.0:1766100171.199938:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ec0c7a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432973824:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.199960:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ec0c7a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432973824:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (225us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100171.199965:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2875
00000100:00100000:2.0:1766100171.200037:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009470f4e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432973824:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100171.200046:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432973696:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100171.200052:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973760
00000100:00100000:3.0:1766100171.200072:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1798
00000100:00100000:3.0:1766100171.200076:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432973760:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766100171.200173:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100171.200204:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432973760:12345-0@lo:8:kworker/u8:2.0 Request processed in 127us (420us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100171.200211:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1798
00000100:00080000:3.0:1766100171.200233:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100171.200236:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100171.200241:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000042e452f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432973760:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766100171.773815:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100172.797792:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100173.821884:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100174.848861:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100175.645021:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100175.645564:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100175.646245:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246144
00000100:00100000:2.0:1766100175.646318:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1946
00000100:00100000:2.0:1766100175.646325:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417246144:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100175.646416:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417246144:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 90us (1409us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100175.646423:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1946
00000100:00100000:3.0:1766100175.646888:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246208
00000100:00100000:3.0:1766100175.646932:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2876
00000100:00100000:3.0:1766100175.646937:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417246208:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100175.646988:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a64620f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417246208:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 50us (1426us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100175.646994:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2876
00000100:00100000:3.0:1766100175.653314:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100175.653451:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246272
00000100:00100000:0.0:1766100175.653541:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1799
00000100:00100000:0.0:1766100175.653549:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417246272:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100175.653653:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100175.653744:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417246272:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 194us (444us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100175.653753:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1799
00010000:00010000:1.0:1766100175.869864:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100176.318809:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100176.318819:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100176.318824:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100176.318834:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4679
00000100:00080000:2.0:1766100176.318838:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100176.318845:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100176.318872:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000063ac9a3f x1851885432974080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100176.318945:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432973952:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319033:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100176.319066:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100176.319072:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432974208:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319103:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100176.319134:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432974016:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319159:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100176.319177:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100176.319181:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432974144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319199:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100176.319231:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974208
00000100:00100000:2.0:1766100176.319278:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1237
00000100:00100000:2.0:1766100176.319286:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432974208:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319343:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432974208:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (239us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100176.319351:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1237
00000100:00100000:2.0:1766100176.319378:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974016
00000100:00100000:2.0:1766100176.319403:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974144
00000100:00100000:2.0:1766100176.319419:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2877
00000100:00100000:2.0:1766100176.319423:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5337db0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432974016:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319457:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5337db0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432974016:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (298us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100176.319465:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2877
00000100:00100000:2.0:1766100176.319503:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2878
00000100:00100000:2.0:1766100176.319507:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432974144:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319547:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e445b90d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432974144:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (347us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100176.319554:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2878
00000100:00100000:2.0:1766100176.319600:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432974144:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100176.319612:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432974016:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766100176.319643:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002ec3fedd x1851885432974208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100183 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766100176.319653:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432974208:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100176.319752:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063ac9a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432974080:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100176.319788:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100176.319889:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432973952
00000100:00100000:0.0:1766100176.319930:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1947
00000100:00100000:0.0:1766100176.319935:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432973952:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100176.319977:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432973952:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (946us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100176.319983:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1947
00000100:00100000:0.0:1766100176.320022:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432973952:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100176.320043:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974080
00000100:00100000:0.0:1766100176.320064:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1800
00000100:00100000:0.0:1766100176.320068:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432974080:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766100176.320173:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100176.320206:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432974080:12345-0@lo:8:kworker/u8:2.0 Request processed in 137us (418us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100176.320213:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1800
00000100:00080000:3.0:1766100176.320388:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100176.320393:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100176.320397:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063ac9a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432974080:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766100176.893769:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100177.918336:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100178.947090:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100179.965726:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100180.770970:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100180.772002:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100180.772057:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100180.772182:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246464
00000100:00100000:1.0:1766100180.772253:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1801
00000100:00100000:1.0:1766100180.772262:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d65d7291 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417246464:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100180.772366:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100180.772446:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d65d7291 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417246464:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 184us (1487us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100180.772456:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1801
00000100:00100000:2.0:1766100180.773371:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246336
00000100:00100000:2.0:1766100180.773446:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1948
00000100:00100000:2.0:1766100180.773452:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417246336:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100180.773565:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417246336:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 112us (1508us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100180.773573:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1948
00000100:00100000:3.0:1766100180.774030:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246400
00000100:00100000:3.0:1766100180.774880:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2879
00000100:00100000:3.0:1766100180.774887:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a02de30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246400:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100180.774938:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a02de30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246400:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 51us (2941us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100180.774945:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2879
00010000:00010000:1.0:1766100180.990894:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100181.438152:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100181.438159:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100181.438162:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1766100181.438306:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432974272:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100181.438534:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100181.439025:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432974336:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100181.439070:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100181.439117:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974336
00000100:00100000:3.0:1766100181.439160:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2880
00000100:00100000:3.0:1766100181.439165:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432974336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100181.439208:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000982e23d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432974336:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (139us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100181.439215:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2880
00000100:00100000:3.0:1766100181.439254:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432974336:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100181.439292:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974272
00000100:00100000:3.0:1766100181.439315:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1949
00000100:00100000:3.0:1766100181.439319:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432974272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100181.439346:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432974272:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (814us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100181.439351:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1949
00000100:00100000:1.0:1766100181.439520:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432974272:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766100181.439618:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4684
00000100:00080000:2.0:1766100181.439621:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100181.439626:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100181.439651:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e485a7e9 x1851885432974400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100181.439735:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432974464:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100181.439768:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100181.439882:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432974400:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100181.439911:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100181.439950:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974400
00000100:00100000:3.0:1766100181.439971:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1802
00000100:00100000:3.0:1766100181.439975:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432974400:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100181.440085:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432974528:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100181.440112:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100181.440154:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974528
00000100:00100000:2.0:1766100181.440181:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1238
00000100:00100000:2.0:1766100181.440185:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432974528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100181.440220:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432974528:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (108us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100181.440226:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1238
00010000:00080000:2.0:1766100181.440255:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000286f9ae8 x1851885432974528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100188 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100181.440264:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432974528:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100181.440282:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974464
00000100:00100000:2.0:1766100181.440303:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2881
00000100:00100000:2.0:1766100181.440307:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432974464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100181.440936:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003523eb7f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432974464:12345-0@lo:400:kworker/u8:3.0 Request processed in 625us (1165us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100181.440943:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2881
00010000:02020000:3.0:1766100181.441356:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100181.441382:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432974464:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100181.441424:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432974400:12345-0@lo:8:kworker/u8:3.0 Request processed in 1446us (1511us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100181.441433:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1802
00000100:00080000:2.0:1766100181.441778:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100181.441783:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100181.441787:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432974400:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766100182.015045:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100183.040775:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100184.061676:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100185.089796:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100185.886938:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100185.887664:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100185.887994:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246528
00000100:00100000:0.0:1766100185.888046:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1950
00000100:00100000:0.0:1766100185.888054:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417246528:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100185.888127:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417246528:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (1197us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100185.888135:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1950
00000100:00100000:2.0:1766100185.888259:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246592
00000100:00100000:2.0:1766100185.888302:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2882
00000100:00100000:2.0:1766100185.888307:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246592:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100185.888342:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a6f4b92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246592:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 34us (681us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100185.888348:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2882
00000100:00100000:3.0:1766100185.888997:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100185.889030:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246656
00000100:00100000:3.0:1766100185.889055:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1803
00000100:00100000:3.0:1766100185.889059:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417246656:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100185.889152:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100185.889200:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417246656:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 140us (205us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100185.889210:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1803
00010000:00010000:2.0:1766100186.111435:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100186.557731:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100186.557739:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100186.557742:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100186.557750:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4689
00000100:00080000:2.0:1766100186.557754:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100186.557765:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100186.557786:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000150fb02 x1851885432974720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100186.557831:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432974656:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.557903:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100186.557935:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100186.557941:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432974784:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.557964:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100186.557989:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432974720:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100186.558035:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100186.558069:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974656
00000100:00100000:2.0:1766100186.558116:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974784
00000100:00100000:2.0:1766100186.558140:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2883
00000100:00100000:2.0:1766100186.558146:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df8193b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432974656:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.558199:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df8193b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432974656:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (296us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100186.558209:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2883
00000100:00100000:2.0:1766100186.558226:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2884
00000100:00100000:2.0:1766100186.558230:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c51db5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432974784:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.558257:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c51db5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432974784:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (293us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100186.558263:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2884
00000100:00100000:2.0:1766100186.558308:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432974784:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.558320:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432974656:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.558364:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974720
00000100:00100000:2.0:1766100186.558386:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1804
00000100:00100000:2.0:1766100186.558390:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432974720:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766100186.558526:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100186.558570:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432974720:12345-0@lo:8:kworker/u8:2.0 Request processed in 180us (535us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100186.558577:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1804
00000100:00100000:0.0:1766100186.558649:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432974848:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100186.558679:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100186.558705:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100186.558710:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432974592:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100186.558734:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100186.558771:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974592
00000100:00100000:0.0:1766100186.558795:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1951
00000100:00100000:0.0:1766100186.558798:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432974592:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100186.558828:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432974592:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (94us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100186.558833:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1951
00000100:00100000:0.0:1766100186.558864:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432974592:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.560905:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974848
00000100:00080000:3.0:1766100186.561067:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100186.561074:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100186.561080:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432974720:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100186.561555:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1239
00000100:00100000:2.0:1766100186.561560:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077d8956b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432974848:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100186.561646:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077d8956b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432974848:12345-0@lo:400:kworker/u8:2.0 Request processed in 84us (2965us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100186.561654:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1239
00010000:00080000:3.0:1766100186.561682:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000096e289f7 x1851885432974848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100193 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766100186.561695:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432974848:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100187.134430:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100188.158269:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100189.182375:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100190.205737:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100191.006607:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100191.006665:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100191.006686:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246784
00000100:00100000:0.0:1766100191.006744:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2885
00000100:00100000:0.0:1766100191.006749:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003a6badc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246784:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100191.006834:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003a6badc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417246784:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 83us (233us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100191.006841:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2885
00000100:00100000:0.0:1766100191.008277:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100191.008373:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246720
00000100:00100000:2.0:1766100191.008608:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246848
00000100:00100000:2.0:1766100191.008642:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1805
00000100:00100000:2.0:1766100191.008648:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417246848:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100191.008749:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100191.008799:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417246848:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 150us (527us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100191.008806:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1805
00000100:00100000:3.0:1766100191.009326:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1952
00000100:00100000:3.0:1766100191.009334:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417246720:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100191.009368:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417246720:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 35us (2704us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100191.009375:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1952
00010000:00010000:1.0:1766100191.229751:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100191.677659:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100191.677666:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100191.677673:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100191.677681:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4695
00000100:00080000:2.0:1766100191.677685:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100191.677690:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100191.677698:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100191.677712:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000096e289f7 x1851885432975040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100191.677714:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432974976:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100191.677790:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100191.677821:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432975104:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100191.677841:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100191.677875:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974976
00000100:00100000:2.0:1766100191.677907:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432975040:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100191.677911:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975104
00000100:00100000:0.0:1766100191.677930:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2886
00000100:00100000:0.0:1766100191.677935:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c8bf3c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432974976:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100191.677938:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100191.677964:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432975168:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100191.677977:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c8bf3c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432974976:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (201us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100191.677985:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2886
00000100:00100000:2.0:1766100191.677985:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100191.677999:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100191.678002:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2887
00000100:00100000:2.0:1766100191.678003:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432974912:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100191.678005:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091b162e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432975104:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100191.678112:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975040
00000100:00100000:1.0:1766100191.678130:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975168
00000100:00100000:1.0:1766100191.678147:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1240
00000100:00100000:1.0:1766100191.678151:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432975168:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100191.678231:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1806
00000100:00100000:3.0:1766100191.678236:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432975040:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766100191.678329:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100191.679227:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091b162e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432975104:12345-0@lo:400:kworker/u8:2.0 Request processed in 1218us (1383us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100191.679431:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2887
00000100:00100000:2.0:1766100191.679465:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100191.679519:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432974912
00000100:00100000:2.0:1766100191.679547:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1953
00000100:00100000:2.0:1766100191.679551:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432974912:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100191.679590:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432974912:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (127us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100191.679596:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1953
00000100:00100000:2.0:1766100191.679717:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432974912:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100191.679875:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432975104:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100191.679880:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432975168:12345-0@lo:400:kworker/u8:2.0 Request processed in 1728us (1895us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100191.679885:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea6f516a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432974976:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100191.679887:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1240
00010000:00080000:1.0:1766100191.679911:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e485a7e9 x1851885432975168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100198 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766100191.679920:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e485a7e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432975168:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100191.680313:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432975040:12345-0@lo:8:kworker/u8:2.0 Request processed in 2075us (2374us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100191.680321:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1806
00000100:00080000:0.0:1766100191.685300:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100191.685309:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100191.685317:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096e289f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432975040:0@lo:8:kworker/u8:2.0
00010000:00010000:2.0:1766100192.253778:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100193.277892:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100194.304283:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100195.326806:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100196.126813:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100196.126858:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100196.126891:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100196.126962:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247040
00000100:00100000:2.0:1766100196.127014:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1807
00000100:00100000:2.0:1766100196.127020:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417247040:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100196.127117:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100196.127167:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417247040:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 147us (361us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100196.127173:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1807
00000100:00100000:1.0:1766100196.127228:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246976
00000100:00100000:1.0:1766100196.127262:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2888
00000100:00100000:1.0:1766100196.127266:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c4f798bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417246976:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100196.127299:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c4f798bf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417246976:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 32us (409us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100196.127305:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2888
00000100:00100000:0.0:1766100196.127446:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417246912
00000100:00100000:0.0:1766100196.127507:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1954
00000100:00100000:0.0:1766100196.127511:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417246912:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.127536:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417246912:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 25us (679us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100196.127542:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1954
00010000:00010000:1.0:1766100196.350473:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100196.799777:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100196.799784:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100196.799788:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100196.799795:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4700
00000100:00080000:0.0:1766100196.799798:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100196.799803:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100196.799822:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e82385d7 x1851885432975360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100196.799862:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000347bd79f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432975296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.799923:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100196.799950:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100196.799955:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432975424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.799975:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100196.800006:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975296
00000100:00100000:0.0:1766100196.800041:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975424
00000100:00100000:0.0:1766100196.800058:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2889
00000100:00100000:0.0:1766100196.800063:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9fe64a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432975296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.800107:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9fe64a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432975296:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (184us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100196.800114:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2889
00000100:00100000:0.0:1766100196.800132:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2890
00000100:00100000:0.0:1766100196.800136:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042e452f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432975424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.800159:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042e452f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432975424:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (184us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100196.800164:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2890
00000100:00100000:0.0:1766100196.800194:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432975424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.800204:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000347bd79f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432975296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.800223:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e82385d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432975360:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100196.800246:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100196.800989:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0a9bb15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432975488:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100196.801027:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100196.801054:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100196.801059:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b46034f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432975232:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100196.801082:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100196.801131:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975360
00000100:00100000:3.0:1766100196.801164:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975488
00000100:00100000:3.0:1766100196.801199:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1241
00000100:00100000:3.0:1766100196.801203:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000edd8a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432975488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100196.801243:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000edd8a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432975488:12345-0@lo:400:kworker/u8:3.0 Request processed in 38us (216us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100196.801249:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1241
00000100:00100000:3.0:1766100196.801263:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1808
00000100:00100000:3.0:1766100196.801266:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000347bd79f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432975360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100196.801360:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100196.801406:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000347bd79f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432975360:12345-0@lo:8:kworker/u8:3.0 Request processed in 138us (1159us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100196.801414:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1808
00000100:00080000:1.0:1766100196.801428:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100196.801434:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100196.801439:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e82385d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432975360:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766100196.801445:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b0a9bb15 x1851885432975488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100203 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100196.801456:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0a9bb15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432975488:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.802432:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975232
00000100:00100000:0.0:1766100196.802500:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1955
00000100:00100000:0.0:1766100196.802505:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432975232:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100196.802557:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432975232:12345-0@lo:400:kworker/u8:3.0 Request processed in 51us (1474us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100196.802563:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1955
00000100:00100000:3.0:1766100196.803509:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b46034f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432975232:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100197.373743:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100198.397870:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100199.421777:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100200.448717:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100201.247507:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100201.249043:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247232
00000100:00100000:2.0:1766100201.250159:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100201.250634:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1809
00000100:00100000:3.0:1766100201.250645:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417247232:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100201.250755:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100201.250841:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417247232:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 196us (3345us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100201.250851:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1809
00000100:00100000:1.0:1766100201.250934:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247104
00000100:00100000:1.0:1766100201.250981:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1956
00000100:00100000:1.0:1766100201.250986:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417247104:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100201.251024:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417247104:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 38us (874us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100201.251031:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1956
00000100:00100000:2.0:1766100201.255576:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100201.256401:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247168
00000100:00100000:2.0:1766100201.256526:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2891
00000100:00100000:2.0:1766100201.256536:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096e289f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417247168:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100201.256652:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096e289f7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417247168:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 116us (1087us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100201.256661:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2891
00010000:00010000:2.0:1766100201.471062:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100201.917939:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100201.917949:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100201.917954:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100201.917965:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4705
00000100:00080000:0.0:1766100201.917969:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100201.917976:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100201.918003:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009470f4e0 x1851885432975680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100201.918058:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432975616:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100201.918141:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100201.918184:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100201.918189:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432975552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100201.918208:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432975744:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100201.918237:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100201.918243:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100201.918272:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100201.918280:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432975808:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100201.918288:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009470f4e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432975680:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100201.918321:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100201.918423:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975552
00000100:00100000:1.0:1766100201.918472:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1957
00000100:00100000:1.0:1766100201.918495:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432975552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100201.918635:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975808
00000100:00100000:0.0:1766100201.918646:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975616
00000100:00100000:3.0:1766100201.918664:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1242
00000100:00100000:3.0:1766100201.918668:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432975808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100201.918854:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975744
00000100:00100000:0.0:1766100201.918874:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2892
00000100:00100000:0.0:1766100201.918878:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007bef605c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432975616:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100201.920862:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432975552:12345-0@lo:400:kworker/u8:2.0 Request processed in 2355us (2614us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100201.920886:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1957
00000100:00100000:2.0:1766100201.920942:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432975552:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100201.921021:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100201.921061:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2893
00000100:00100000:1.0:1766100201.921071:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c6056fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432975744:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100201.921113:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975680
00000100:00100000:2.0:1766100201.921149:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1810
00000100:00100000:2.0:1766100201.921153:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea6f516a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432975680:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:2.0:1766100201.921251:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100201.921283:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c6056fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432975744:12345-0@lo:400:kworker/u8:2.0 Request processed in 212us (3041us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100201.921293:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2893
00000100:00100000:2.0:1766100201.921302:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea6f516a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432975680:12345-0@lo:8:kworker/u8:2.0 Request processed in 146us (280us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100201.921312:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1810
00000100:00100000:1.0:1766100201.921370:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432975744:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766100201.924523:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100201.924548:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100201.924558:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009470f4e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432975680:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100201.924849:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007bef605c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432975616:12345-0@lo:400:kworker/u8:2.0 Request processed in 5967us (6707us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100201.924864:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2892
00000100:00100000:2.0:1766100201.924981:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432975616:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100201.925066:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432975808:12345-0@lo:400:kworker/u8:2.0 Request processed in 6395us (6745us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100201.925078:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1242
00010000:00080000:1.0:1766100201.925090:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000055153256 x1851885432975808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100208 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766100201.925103:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432975808:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100202.493824:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100203.518874:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100204.541844:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100205.569820:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100206.365148:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100206.365202:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100206.365242:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247360
00000100:00100000:0.0:1766100206.365357:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2894
00000100:00100000:0.0:1766100206.365367:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417247360:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100206.365519:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a108df5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417247360:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 152us (382us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100206.365528:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2894
00000100:00100000:2.0:1766100206.366583:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247296
00000100:00100000:2.0:1766100206.366658:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1958
00000100:00100000:2.0:1766100206.366665:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417247296:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100206.366966:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417247296:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 300us (1764us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100206.367132:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1958
00000100:00100000:0.0:1766100206.367401:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100206.368168:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247424
00000100:00100000:3.0:1766100206.368227:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1811
00000100:00100000:3.0:1766100206.368233:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417247424:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100206.369179:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100206.369284:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417247424:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1049us (1884us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100206.369296:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1811
00010000:00010000:1.0:1766100206.597237:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100207.037845:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1766100207.037846:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:0.0:1766100207.037855:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100207.037860:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100207.037872:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4710
00000100:00080000:0.0:1766100207.037876:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:3.0:1766100207.037878:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432975872:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100207.037886:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100207.037916:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009470f4e0 x1851885432976000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100207.037970:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432976064:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100207.037977:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100207.038017:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432976128:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100207.038018:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100207.038045:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100207.038052:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432975936:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100207.038058:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100207.038109:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975872
00000100:00100000:2.0:1766100207.038127:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009470f4e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976000:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100207.038165:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1959
00000100:00100000:3.0:1766100207.038177:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432975872:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100207.038182:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100207.038224:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976064
00000100:00100000:3.0:1766100207.038256:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432975872:12345-0@lo:400:kworker/u8:3.0 Request processed in 82us (285us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100207.038265:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2895
00000100:00100000:3.0:1766100207.038268:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1959
00000100:00100000:2.0:1766100207.038272:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46034f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432976064:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100207.038533:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001047d83 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432975872:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100207.038539:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46034f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432976064:12345-0@lo:400:kworker/u8:3.0 Request processed in 266us (523us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100207.038546:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2895
00000100:00100000:3.0:1766100207.038571:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976128
00000100:00100000:2.0:1766100207.038574:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976000
00000100:00100000:2.0:1766100207.038595:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1243
00000100:00100000:2.0:1766100207.038601:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432976128:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100207.038607:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1812
00000100:00100000:3.0:1766100207.038613:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088496005 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432976000:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100207.038633:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432976128:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (576us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100207.038641:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1243
00010000:00080000:2.0:1766100207.038944:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001415cdc3 x1851885432976128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100214 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100207.038957:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001415cdc3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432976128:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766100207.038963:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100207.039012:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088496005 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432976000:12345-0@lo:8:kworker/u8:3.0 Request processed in 397us (836us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100207.039021:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1812
00000100:00080000:2.0:1766100207.039024:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100207.039030:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100207.039034:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009470f4e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976000:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100207.044701:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100207.044793:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c0fbdcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432976064:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100207.045006:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432975936
00000100:00100000:1.0:1766100207.048055:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2896
00000100:00100000:1.0:1766100207.048072:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0a9bb15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432975936:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100207.048209:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0a9bb15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432975936:12345-0@lo:400:kworker/u8:3.0 Request processed in 138us (3519us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100207.048908:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2896
00000100:00100000:0.0:1766100207.051316:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055153256 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432975936:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100207.613931:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100208.638151:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100209.663304:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100210.687828:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100211.486907:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100211.486964:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100211.487003:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100211.487049:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247552
00000100:00100000:2.0:1766100211.487128:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2897
00000100:00100000:2.0:1766100211.487136:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417247552:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100211.487227:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5be2c2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417247552:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 91us (263us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100211.487237:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2897
00000100:00100000:1.0:1766100211.487612:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247616
00000100:00100000:0.0:1766100211.487624:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247488
00000100:00100000:0.0:1766100211.487663:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1960
00000100:00100000:0.0:1766100211.487668:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417247488:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100211.487700:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417247488:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 33us (807us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100211.487707:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1960
00000100:00100000:1.0:1766100211.487857:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1813
00000100:00100000:1.0:1766100211.487863:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417247616:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100211.487966:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100211.487999:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084112edb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417247616:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 135us (996us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100211.488009:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1813
00010000:00010000:1.0:1766100211.712548:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100212.158885:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100212.158894:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100212.158898:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100212.158909:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4715
00000100:00080000:2.0:1766100212.158912:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100212.158927:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100212.158953:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000acd0bfef x1851885432976320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100212.159000:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432976256:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100212.159082:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100212.159113:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100212.159117:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078059cfe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432976384:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100212.159136:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100212.159157:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acd0bfef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976320:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100212.159180:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100212.159216:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976256
00000100:00100000:2.0:1766100212.159288:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976384
00000100:00100000:2.0:1766100212.159308:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2898
00000100:00100000:2.0:1766100212.159314:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432976256:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100212.159364:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6716e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432976256:12345-0@lo:400:kworker/u8:3.0 Request processed in 50us (284us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100212.159372:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2898
00000100:00100000:2.0:1766100212.159389:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2899
00000100:00100000:2.0:1766100212.159392:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432976384:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100212.159411:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063ac9a3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432976384:12345-0@lo:400:kworker/u8:3.0 Request processed in 18us (275us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100212.159416:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2899
00000100:00100000:2.0:1766100212.159448:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078059cfe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432976384:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100212.159456:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432976256:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100212.159655:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040addeb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432976448:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100212.159685:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100212.159705:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100212.159709:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432976192:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100212.159733:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100212.159764:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976320
00000100:00100000:0.0:1766100212.159783:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976448
00000100:00100000:0.0:1766100212.159803:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1244
00000100:00100000:0.0:1766100212.159806:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432976448:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100212.159836:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432976448:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (151us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100212.159842:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1244
00000100:00100000:0.0:1766100212.159854:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1814
00000100:00100000:0.0:1766100212.159858:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432976320:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100212.159958:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100212.159985:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432976320:12345-0@lo:8:kworker/u8:3.0 Request processed in 126us (805us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100212.159990:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1814
00010000:00080000:0.0:1766100212.160012:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000040addeb2 x1851885432976448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100219 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100212.160021:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040addeb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432976448:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100212.160040:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100212.160044:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100212.160047:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acd0bfef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976320:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100212.160064:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976192
00000100:00100000:0.0:1766100212.160084:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1961
00000100:00100000:0.0:1766100212.160088:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432976192:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100212.160113:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432976192:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (380us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100212.160118:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1961
00000100:00100000:0.0:1766100212.160145:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000150fb02 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432976192:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100212.736927:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100213.757944:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100214.797820:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100215.805716:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100216.606406:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100216.606549:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247680
00000100:00100000:0.0:1766100216.606627:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1962
00000100:00100000:0.0:1766100216.606636:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417247680:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100216.606738:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417247680:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 102us (345us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100216.606748:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1962
00000100:00100000:0.0:1766100216.612369:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100216.612430:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100216.612536:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247808
00000100:00100000:1.0:1766100216.612569:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247744
00000100:00100000:2.0:1766100216.612597:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1815
00000100:00100000:2.0:1766100216.612605:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e82385d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417247808:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:1.0:1766100216.612622:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2900
00000100:00100000:1.0:1766100216.612626:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003c0fbdcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417247744:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00010000:02020000:2.0:1766100216.612707:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100216.612724:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003c0fbdcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417247744:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 96us (369us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100216.612732:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2900
00000100:00100000:2.0:1766100216.612739:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e82385d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417247808:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 134us (311us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100216.612745:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1815
00010000:00010000:0.0:1766100216.830170:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100217.287927:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100217.287937:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100217.287941:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100217.287952:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4720
00000100:00080000:0.0:1766100217.287956:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100217.287964:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100217.288003:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e446c544 x1851885432976640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100217.288084:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001238be68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432976704:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100217.288186:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100217.288236:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100217.288242:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003e40221d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432976576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100217.288265:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100217.288321:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e446c544 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976640:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100217.288353:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100217.288404:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976704
00000100:00100000:0.0:1766100217.288470:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976576
00000100:00100000:0.0:1766100217.288514:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2901
00000100:00100000:0.0:1766100217.288521:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d65d7291 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432976704:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100217.288597:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d65d7291 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432976704:12345-0@lo:400:kworker/u8:2.0 Request processed in 76us (413us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100217.288605:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2901
00000100:00100000:0.0:1766100217.288643:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2902
00000100:00100000:0.0:1766100217.288647:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077d8956b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432976576:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100217.288672:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077d8956b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432976576:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (407us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100217.288677:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2902
00000100:00100000:0.0:1766100217.288697:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976640
00000100:00100000:0.0:1766100217.288713:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1816
00000100:00100000:0.0:1766100217.288717:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432976640:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766100217.288841:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100217.288882:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432976640:12345-0@lo:8:kworker/u8:2.0 Request processed in 164us (530us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100217.288888:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1816
00000100:00100000:0.0:1766100217.288944:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003e40221d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432976576:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100217.288960:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001238be68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432976704:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766100217.288994:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100217.288997:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100217.289001:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e446c544 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976640:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100217.289452:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d45f753 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432976768:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100217.289536:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100217.289569:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100217.289576:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000617de322 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432976512:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100217.289618:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100217.289666:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976768
00000100:00100000:2.0:1766100217.289717:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1245
00000100:00100000:2.0:1766100217.289721:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432976768:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100217.289759:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432976768:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (224us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100217.289766:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1245
00000100:00100000:2.0:1766100217.289820:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976512
00000100:00100000:2.0:1766100217.289846:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1963
00000100:00100000:2.0:1766100217.289850:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432976512:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100217.289879:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432976512:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (262us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100217.289886:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1963
00000100:00100000:2.0:1766100217.289925:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000617de322 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432976512:0@lo:400:kworker/u8:2.0
00010000:00080000:2.0:1766100217.289937:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002d45f753 x1851885432976768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100224 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766100217.289947:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d45f753 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432976768:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100217.858016:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100218.879050:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100219.908655:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100220.929677:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100221.729601:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100221.729689:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100221.729846:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247936
00000100:00100000:0.0:1766100221.730521:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417247872
00000100:00100000:0.0:1766100221.730600:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1964
00000100:00100000:0.0:1766100221.730611:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417247872:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100221.730746:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417247872:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 136us (1057us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100221.730759:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1964
00000100:00100000:2.0:1766100221.730783:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2903
00000100:00100000:2.0:1766100221.730804:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417247936:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100221.732078:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100221.732249:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1307ce3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417247936:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 1076us (2291us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100221.732264:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2903
00000100:00100000:1.0:1766100221.733558:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248000
00000100:00100000:1.0:1766100221.733652:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1817
00000100:00100000:1.0:1766100221.733662:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417248000:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100221.733777:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100221.733877:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417248000:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 216us (1805us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100221.733886:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1817
00010000:00010000:1.0:1766100221.949848:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100222.398698:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100222.398708:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100222.398714:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100222.398727:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4725
00000100:00080000:1.0:1766100222.398731:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100222.398739:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100222.398769:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432976896:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100222.398778:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000030ebde24 x1851885432976960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100222.398863:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432977024:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100222.398874:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100222.398943:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100222.398986:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043119846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432977088:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100222.399014:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976960:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100222.399046:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100222.399090:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976896
00000100:00100000:3.0:1766100222.399115:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100222.399131:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432976832:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100222.399165:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977024
00000100:00100000:1.0:1766100222.399228:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2904
00000100:00100000:3.0:1766100222.399228:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100222.399245:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432976896:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100222.399314:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976832
00000100:00100000:1.0:1766100222.399330:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f63adac5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432976896:12345-0@lo:400:kworker/u8:3.0 Request processed in 87us (470us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100222.399345:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2904
00000100:00100000:1.0:1766100222.399385:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2905
00000100:00100000:1.0:1766100222.399392:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b510690 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432977024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100222.399402:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1965
00000100:00100000:3.0:1766100222.399417:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432976832:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100222.399441:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b510690 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432977024:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (502us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100222.399464:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2905
00000100:00100000:1.0:1766100222.399564:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432977024:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100222.399568:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432976832:12345-0@lo:400:kworker/u8:3.0 Request processed in 149us (346us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100222.399587:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1965
00000100:00100000:3.0:1766100222.399665:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977088
00000100:00100000:3.0:1766100222.399708:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1246
00000100:00100000:3.0:1766100222.399713:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432977088:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100222.399726:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100222.399775:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099002e71 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432977088:12345-0@lo:400:kworker/u8:3.0 Request processed in 60us (737us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100222.399783:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1246
00000100:00100000:1.0:1766100222.399792:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432976896:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100222.399800:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432976960
00000100:00100000:3.0:1766100222.399827:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1818
00000100:00100000:3.0:1766100222.399830:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432976960:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100222.399932:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100222.399970:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432976960:12345-0@lo:8:kworker/u8:3.0 Request processed in 139us (256us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100222.399976:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1818
00000100:00080000:1.0:1766100222.406216:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100222.406237:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100222.406250:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432976960:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100222.406254:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432976832:0@lo:400:kworker/u8:3.0
00010000:00080000:0.0:1766100222.406281:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000043119846 x1851885432977088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100229 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100222.406294:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043119846 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432977088:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100222.976287:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100223.997779:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100225.023220:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100226.045869:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100226.847106:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100226.849592:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100226.850106:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100226.850188:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248128
00000100:00100000:2.0:1766100226.850276:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2906
00000100:00100000:2.0:1766100226.850285:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000edd8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417248128:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100226.850391:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000edd8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417248128:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 106us (287us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100226.850399:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2906
00000100:00100000:0.0:1766100226.850862:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248192
00000100:00100000:0.0:1766100226.850901:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1819
00000100:00100000:0.0:1766100226.850906:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417248192:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100226.851011:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100226.852002:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417248192:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1090us (4906us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100226.852020:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1819
00000100:00100000:1.0:1766100226.852089:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248064
00000100:00100000:1.0:1766100226.852159:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1966
00000100:00100000:1.0:1766100226.852166:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417248064:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100226.852229:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417248064:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 63us (2651us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100226.852237:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1966
00010000:00010000:1.0:1766100227.069760:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100227.518060:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100227.518072:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100227.518076:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100227.518089:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4730
00000100:00080000:1.0:1766100227.518095:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100227.518102:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100227.518153:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000678c16b3 x1851885432977280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100227.518215:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432977344:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100227.518306:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100227.518356:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100227.518361:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432977216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100227.518406:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100227.518437:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432977280:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100227.518443:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a75d9f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432977408:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100227.518500:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100227.518739:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432977152:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100227.518911:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977280
00000100:00100000:2.0:1766100227.518957:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1820
00000100:00100000:2.0:1766100227.519465:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000914f119b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432977280:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100227.519622:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100227.519639:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977344
00000100:00100000:1.0:1766100227.519695:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977216
00000100:00100000:0.0:1766100227.519711:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977408
00000100:00100000:1.0:1766100227.519720:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2907
00000100:00100000:1.0:1766100227.519725:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a48ec222 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432977344:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100227.519748:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1247
00000100:00100000:0.0:1766100227.519753:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432977408:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100227.519796:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a48ec222 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432977344:12345-0@lo:400:kworker/u8:3.0 Request processed in 69us (1491us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100227.519802:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432977408:12345-0@lo:400:kworker/u8:3.0 Request processed in 47us (183us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100227.519806:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2907
00000100:00100000:0.0:1766100227.519814:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1247
00000100:00100000:1.0:1766100227.519838:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2908
00000100:00100000:1.0:1766100227.519843:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000044763d3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432977216:12345-0@lo:400:kworker/u8:3.0
00010000:00080000:0.0:1766100227.519869:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000006a75d9f1 x1851885432977408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100234 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100227.519879:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000044763d3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432977216:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (1471us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100227.519887:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2908
00000100:00100000:0.0:1766100227.519890:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a75d9f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432977408:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100227.519977:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432977216:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100227.519990:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432977344:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100227.520976:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:02020000:2.0:1766100227.521777:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100227.521858:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000914f119b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432977280:12345-0@lo:8:kworker/u8:3.0 Request processed in 2393us (3381us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100227.521871:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1820
00000100:00080000:2.0:1766100227.521960:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100227.521964:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100227.521968:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432977280:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100227.528775:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977152
00000100:00100000:0.0:1766100227.528858:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1967
00000100:00100000:0.0:1766100227.528865:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432977152:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100227.528996:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432977152:12345-0@lo:400:kworker/u8:3.0 Request processed in 130us (8023us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100227.529004:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1967
00000100:00100000:2.0:1766100227.529841:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432977152:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100228.094403:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100229.120030:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100230.142465:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100231.165741:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100231.969405:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100231.969457:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100231.969473:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248256
00000100:00100000:1.0:1766100231.970617:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1968
00000100:00100000:1.0:1766100231.970627:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417248256:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100231.971676:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417248256:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 1045us (2275us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100231.971686:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1968
00000100:00100000:0.0:1766100231.971695:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248320
00000100:00100000:0.0:1766100231.971756:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2909
00000100:00100000:0.0:1766100231.971762:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417248320:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100231.971978:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf96017c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417248320:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 215us (2522us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100231.971990:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2909
00000100:00100000:3.0:1766100231.972334:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100231.972421:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248384
00000100:00100000:0.0:1766100231.973085:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1821
00000100:00100000:0.0:1766100231.973092:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417248384:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100231.973195:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100231.973260:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417248384:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 168us (927us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100231.973267:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1821
00010000:00010000:0.0:1766100232.190184:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100232.637856:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766100232.637858:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:1.0:1766100232.637866:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100232.637869:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100232.637877:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4735
00000100:00080000:1.0:1766100232.637881:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:2.0:1766100232.637882:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432977472:0@lo:400:kworker/u8:2.0
00000100:00080000:1.0:1766100232.637888:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100232.637911:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004dffa5d1 x1851885432977600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100232.637960:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432977664:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100232.637961:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100232.637997:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432977728:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100232.638004:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100232.638027:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100232.638034:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432977536:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100232.638050:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100232.638073:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100232.638091:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432977600:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766100232.638124:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977664
00000100:00100000:2.0:1766100232.638134:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100232.638166:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977536
00000100:00100000:2.0:1766100232.638174:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977472
00000100:00100000:2.0:1766100232.638207:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1969
00000100:00100000:2.0:1766100232.638213:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432977472:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100232.638258:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432977472:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (301us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100232.638265:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1969
00000100:00100000:2.0:1766100232.638298:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977728
00000100:00100000:2.0:1766100232.638318:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977600
00000100:00100000:2.0:1766100232.638333:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1248
00000100:00100000:2.0:1766100232.638338:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432977728:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100232.638374:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432977728:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (339us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100232.638380:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1248
00000100:00100000:2.0:1766100232.638398:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1822
00000100:00100000:2.0:1766100232.638403:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432977600:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100232.638412:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2910
00000100:00100000:0.0:1766100232.638418:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a75d9f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432977664:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100232.638450:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2911
00000100:00100000:1.0:1766100232.638455:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043119846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432977536:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100232.638470:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a75d9f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432977664:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (465us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100232.638501:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2910
00000100:00100000:1.0:1766100232.638514:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043119846 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432977536:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (441us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100232.638521:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2911
00010000:02020000:2.0:1766100232.638524:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100232.638556:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0ded71f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432977536:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100232.638566:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432977600:12345-0@lo:8:kworker/u8:2.0 Request processed in 162us (432us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100232.638575:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432977664:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100232.638576:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1822
00000100:00080000:2.0:1766100232.638616:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100232.638620:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100232.638624:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432977600:0@lo:8:kworker/u8:2.0
00010000:00080000:0.0:1766100232.638629:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000678c16b3 x1851885432977728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100239 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766100232.638640:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432977728:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100232.638652:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432977472:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100233.215903:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100234.237711:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100235.262330:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100236.285782:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100237.085736:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100237.085816:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100237.085830:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248512
00000100:00100000:1.0:1766100237.085906:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2912
00000100:00100000:1.0:1766100237.085913:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea6f516a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417248512:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100237.086018:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea6f516a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417248512:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 105us (290us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100237.086026:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2912
00000100:00100000:3.0:1766100237.086170:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248448
00000100:00100000:3.0:1766100237.086231:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1970
00000100:00100000:3.0:1766100237.086238:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417248448:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100237.086310:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417248448:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 71us (497us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100237.086318:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1970
00000100:00100000:2.0:1766100237.090509:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100237.090979:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248576
00000100:00100000:3.0:1766100237.091046:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1823
00000100:00100000:3.0:1766100237.091053:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417248576:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100237.091155:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100237.091241:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417248576:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 186us (742us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100237.091249:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1823
00010000:00010000:1.0:1766100237.309729:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100237.757878:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100237.757885:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100237.757888:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100237.757896:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4741
00000100:00080000:1.0:1766100237.757898:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100237.757904:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100237.757924:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000678c16b3 x1851885432977920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100237.757964:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432977856:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758027:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100237.758057:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100237.758063:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432977984:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758101:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100237.758125:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432977920:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758150:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100237.758183:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977856
00000100:00100000:2.0:1766100237.758214:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432978048:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758218:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977984
00000100:00100000:1.0:1766100237.758237:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2913
00000100:00100000:1.0:1766100237.758242:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432977856:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100237.758242:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100237.758257:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100237.758262:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432977792:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758285:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0ded71f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432977856:12345-0@lo:400:kworker/u8:3.0 Request processed in 42us (259us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100237.758286:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100237.758292:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2913
00000100:00100000:1.0:1766100237.758306:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2914
00000100:00100000:2.0:1766100237.758308:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977792
00000100:00100000:1.0:1766100237.758310:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432977984:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100237.758329:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1971
00000100:00100000:2.0:1766100237.758333:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432977792:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758334:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f25a79b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432977984:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (233us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100237.758339:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2914
00000100:00100000:1.0:1766100237.758359:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432977920
00000100:00100000:2.0:1766100237.758361:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432977792:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (76us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100237.758367:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1971
00000100:00100000:1.0:1766100237.758374:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978048
00000100:00100000:1.0:1766100237.758391:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1249
00000100:00100000:1.0:1766100237.758394:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432978048:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100237.758403:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000050b9c6b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432977792:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758420:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432978048:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (178us total) trans 0 rc 0/0
00010000:00080000:2.0:1766100237.758422:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000030ebde24 x1851885432978048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100244 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100237.758425:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1249
00000100:00100000:2.0:1766100237.758432:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432978048:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758434:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1824
00000100:00100000:1.0:1766100237.758438:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432977920:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100237.758552:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100237.758592:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432977920:12345-0@lo:8:kworker/u8:3.0 Request processed in 153us (441us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100237.758598:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1824
00000100:00100000:1.0:1766100237.758620:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004dffa5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432977984:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100237.758629:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad5bbc91 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432977856:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766100237.758762:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100237.758766:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100237.758770:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432977920:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766100238.333676:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100239.363498:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100240.381713:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100241.406001:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100242.205021:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100242.205107:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248768
00000100:00100000:3.0:1766100242.205156:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1825
00000100:00100000:3.0:1766100242.205164:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417248768:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100242.205258:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100242.205329:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417248768:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 165us (315us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100242.205339:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1825
00000100:00100000:2.0:1766100242.205812:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100242.206123:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248704
00000100:00100000:2.0:1766100242.206139:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100242.206199:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248640
00000100:00100000:0.0:1766100242.206219:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2915
00000100:00100000:0.0:1766100242.206225:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040addeb2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417248704:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100242.206242:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1972
00000100:00100000:3.0:1766100242.206248:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417248640:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100242.206277:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040addeb2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417248704:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 50us (466us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100242.206284:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417248640:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 34us (144us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100242.206286:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2915
00000100:00100000:3.0:1766100242.206295:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1972
00010000:00010000:1.0:1766100242.430740:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100242.877691:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100242.877698:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100242.877701:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100242.877709:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4746
00000100:00080000:1.0:1766100242.877712:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100242.877718:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100242.877737:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000da5bbb9 x1851885432978240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100242.877872:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432978368:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.877952:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100242.877987:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100242.877992:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432978112:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878018:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100242.878071:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432978304:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878098:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100242.878114:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100242.878119:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432978176:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878138:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100242.878156:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432978240:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878182:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100242.878214:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978112
00000100:00100000:2.0:1766100242.878249:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1973
00000100:00100000:2.0:1766100242.878254:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432978112:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878294:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432978112:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (275us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100242.878300:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1973
00000100:00100000:2.0:1766100242.878343:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978304
00000100:00100000:2.0:1766100242.878363:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978176
00000100:00100000:2.0:1766100242.878375:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2916
00000100:00100000:2.0:1766100242.878379:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad5bbc91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432978304:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878410:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad5bbc91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432978304:12345-0@lo:400:kworker/u8:3.0 Request processed in 30us (312us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100242.878415:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2916
00000100:00100000:2.0:1766100242.878426:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2917
00000100:00100000:2.0:1766100242.878429:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004dffa5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432978176:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878449:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004dffa5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432978176:12345-0@lo:400:kworker/u8:3.0 Request processed in 19us (311us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100242.878454:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2917
00000100:00100000:2.0:1766100242.878505:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432978176:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878517:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432978304:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100242.878541:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432978112:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100242.883427:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978368
00000100:00100000:3.0:1766100242.883512:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978240
00000100:00100000:3.0:1766100242.883550:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1250
00000100:00100000:3.0:1766100242.883557:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000678c16b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432978368:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100242.883651:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000678c16b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432978368:12345-0@lo:400:kworker/u8:3.0 Request processed in 94us (5700us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100242.883659:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1250
00000100:00100000:3.0:1766100242.883676:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1826
00000100:00100000:3.0:1766100242.883679:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432978240:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100242.883766:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100242.883798:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ebde24 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432978240:12345-0@lo:8:kworker/u8:3.0 Request processed in 118us (5616us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100242.883804:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1826
00000100:00080000:3.0:1766100242.883849:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100242.883853:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100242.883857:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432978240:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766100242.883892:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000362df469 x1851885432978368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100249 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100242.883902:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000362df469 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432978368:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100243.455405:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100244.477714:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100245.501798:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100246.532271:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100247.326939:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100247.327032:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100247.327070:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248960
00000100:00100000:1.0:1766100247.327105:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100247.327143:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1827
00000100:00100000:2.0:1766100247.327157:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417248960:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00000100:00100000:1.0:1766100247.327180:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248896
00010000:02020000:2.0:1766100247.327256:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100247.327280:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2918
00000100:00100000:1.0:1766100247.327301:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000362df469 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417248896:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100247.327333:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417248960:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 180us (414us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100247.327343:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1827
00000100:00100000:1.0:1766100247.327433:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000362df469 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417248896:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 119us (331us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100247.327443:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2918
00000100:00100000:0.0:1766100247.327612:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417248832
00000100:00100000:0.0:1766100247.327662:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1974
00000100:00100000:0.0:1766100247.327668:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417248832:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100247.328452:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417248832:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 780us (1417us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100247.328461:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1974
00010000:00010000:0.0:1766100247.550663:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100247.997908:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100247.997919:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100247.997924:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100247.997936:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4751
00000100:00080000:1.0:1766100247.997940:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100247.997948:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100247.997975:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b3eafafb x1851885432978560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100248.001162:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432978432:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100248.001162:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432978496:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100248.001278:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432978560:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100248.004143:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432978624:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100248.005181:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100248.005264:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000074406767 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432978688:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100248.005300:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100248.005368:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978560
00000100:00100000:3.0:1766100248.005412:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978688
00000100:00100000:3.0:1766100248.005443:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1251
00000100:00100000:3.0:1766100248.005450:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432978688:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100248.005534:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432978688:12345-0@lo:400:kworker/u8:3.0 Request processed in 83us (233us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100248.005543:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1251
00000100:00100000:3.0:1766100248.005566:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1828
00000100:00100000:3.0:1766100248.005569:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432978560:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100248.005670:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100248.005705:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a851b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432978560:12345-0@lo:8:kworker/u8:3.0 Request processed in 134us (531us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100248.005711:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1828
00010000:00080000:3.0:1766100248.005760:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000074406767 x1851885432978688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100255 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100248.005775:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000074406767 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432978688:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100248.005807:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100248.005860:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978432
00000100:00100000:0.0:1766100248.005894:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1975
00000100:00100000:0.0:1766100248.005899:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432978432:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100248.005934:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432978432:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (127us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100248.005941:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1975
00000100:00100000:1.0:1766100248.005944:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100248.005980:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000da5bbb9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432978432:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100248.006031:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978624
00000100:00100000:0.0:1766100248.006060:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2919
00000100:00100000:0.0:1766100248.006064:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b44b2785 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432978624:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100248.006098:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b44b2785 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432978624:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (154us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100248.006104:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2919
00000100:00100000:0.0:1766100248.006145:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432978624:0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766100248.006335:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100248.006340:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100248.006344:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3eafafb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432978560:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100248.006461:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100248.006619:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978496
00000100:00100000:1.0:1766100248.006693:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2920
00000100:00100000:1.0:1766100248.006698:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000050b9c6b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432978496:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100248.007200:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000050b9c6b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432978496:12345-0@lo:400:kworker/u8:3.0 Request processed in 501us (739us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100248.007236:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aea833b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432978496:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100248.007511:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2920
00010000:00010000:1.0:1766100248.577758:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100249.597783:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100250.624033:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100251.646104:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100252.456028:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100252.456075:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100252.457264:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100252.457325:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249024
00000100:00100000:2.0:1766100252.457349:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249088
00000100:00100000:0.0:1766100252.457381:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1976
00000100:00100000:2.0:1766100252.457381:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2921
00000100:00100000:0.0:1766100252.457387:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249024:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100252.457388:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417249088:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100252.457456:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249024:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 68us (1380us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100252.457463:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1976
00000100:00100000:2.0:1766100252.457937:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009db79f91 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417249088:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 549us (674us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100252.457944:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2921
00000100:00100000:3.0:1766100252.459228:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249152
00000100:00100000:3.0:1766100252.465171:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1829
00000100:00100000:3.0:1766100252.465180:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417249152:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100252.465273:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100252.465340:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417249152:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 159us (9319us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100252.465348:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1829
00010000:00010000:1.0:1766100252.670071:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100253.119826:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100253.119833:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100253.119836:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100253.119843:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4756
00000100:00080000:2.0:1766100253.119846:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:0.0:1766100253.119851:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce3f3b2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432978752:0@lo:400:kworker/u8:2.0
00000100:00080000:2.0:1766100253.119852:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100253.119870:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000ba0acca1 x1851885432978880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100253.119909:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba0acca1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432978880:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100253.119930:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100253.119937:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100253.119956:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100253.119974:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432979008:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100253.119989:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978880
00000100:00100000:0.0:1766100253.119995:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100253.120015:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432978944:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100253.120017:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979008
00000100:00100000:0.0:1766100253.120039:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100253.120041:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1252
00000100:00100000:2.0:1766100253.120047:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432979008:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100253.120054:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100253.120058:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cb80abc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432978816:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100253.120076:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100253.120087:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432979008:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (92us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100253.120095:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1252
00000100:00100000:0.0:1766100253.120098:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978752
00000100:00100000:2.0:1766100253.120109:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1830
00000100:00100000:2.0:1766100253.120112:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432978880:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100253.120119:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1977
00000100:00100000:0.0:1766100253.120121:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432978752:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100253.120147:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432978752:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (219us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100253.120152:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1977
00000100:00100000:0.0:1766100253.120178:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978944
00000100:00100000:0.0:1766100253.120197:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432978816
00010000:02020000:2.0:1766100253.120203:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100253.120229:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2922
00000100:00100000:2.0:1766100253.120230:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432978880:12345-0@lo:8:kworker/u8:2.0 Request processed in 117us (294us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100253.120232:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e40221d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432978944:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100253.120236:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1830
00000100:00080000:2.0:1766100253.120268:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100253.120271:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100253.120275:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba0acca1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432978880:0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100253.120329:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2923
00000100:00100000:2.0:1766100253.120333:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000347bd79f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432978816:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100253.120883:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e40221d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432978944:12345-0@lo:400:kworker/u8:2.0 Request processed in 648us (842us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100253.120890:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2922
00010000:00080000:0.0:1766100253.120915:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000fc68c7d9 x1851885432979008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100260 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766100253.120924:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432979008:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100253.120931:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce3f3b2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432978752:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100253.121196:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000347bd79f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432978816:12345-0@lo:400:kworker/u8:2.0 Request processed in 861us (1118us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100253.121199:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432978944:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100253.121202:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2923
00000100:00100000:1.0:1766100253.121214:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cb80abc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432978816:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766100253.694613:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100254.717998:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100255.742645:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100256.765777:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100257.564923:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100257.564975:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100257.565008:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100257.565110:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249216
00000100:00100000:0.0:1766100257.565160:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1978
00000100:00100000:0.0:1766100257.565167:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249216:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100257.565233:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249216:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 66us (318us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100257.565240:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1978
00000100:00100000:2.0:1766100257.565433:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249280
00000100:00100000:2.0:1766100257.565468:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2924
00000100:00100000:2.0:1766100257.565499:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417249280:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100257.565543:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ce9aa68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417249280:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 70us (569us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100257.565550:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2924
00000100:00100000:3.0:1766100257.565703:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249344
00000100:00100000:3.0:1766100257.565732:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1831
00000100:00100000:3.0:1766100257.565735:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417249344:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100257.565827:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100257.566378:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417249344:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 640us (1368us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100257.566385:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1831
00010000:00010000:1.0:1766100257.790409:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100258.238075:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba0acca1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432979072:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100258.238156:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766100258.238240:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100258.238245:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100258.238248:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100258.238252:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4761
00000100:00080000:2.0:1766100258.238255:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100258.238260:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100258.238281:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000f12cf851 x1851885432979200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100258.238712:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432979328:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.238749:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100258.238827:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979328
00000100:00100000:3.0:1766100258.238870:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1253
00000100:00100000:3.0:1766100258.238876:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432979328:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.238933:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432979328:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (171us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100258.238941:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1253
00010000:00080000:3.0:1766100258.238977:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ff2504ac x1851885432979328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100265 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766100258.238987:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff2504ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432979328:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239020:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432979136:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239046:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100258.239062:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100258.239067:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cb80abc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432979264:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239086:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100258.239109:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979136
00000100:00100000:3.0:1766100258.239130:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979264
00000100:00100000:3.0:1766100258.239144:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2925
00000100:00100000:3.0:1766100258.239147:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432979136:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239178:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4e5a5c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432979136:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (133us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100258.239184:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2925
00000100:00100000:3.0:1766100258.239195:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2926
00000100:00100000:3.0:1766100258.239198:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432979264:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239221:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057275e9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432979264:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (135us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100258.239226:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2926
00000100:00100000:3.0:1766100258.239246:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cb80abc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432979264:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239254:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe6e47c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432979136:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239279:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432979200:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100258.239301:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100258.239323:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979200
00000100:00100000:3.0:1766100258.239338:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1832
00000100:00100000:3.0:1766100258.239341:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432979200:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766100258.239429:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100258.239459:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432979200:12345-0@lo:8:kworker/u8:2.0 Request processed in 117us (158us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100258.239465:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1832
00000100:00080000:3.0:1766100258.239505:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100258.239509:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100258.239512:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432979200:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766100258.239832:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979072
00000100:00100000:1.0:1766100258.239867:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1979
00000100:00100000:1.0:1766100258.239871:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432979072:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100258.242340:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432979072:12345-0@lo:400:kworker/u8:2.0 Request processed in 2461us (4178us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100258.242356:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1979
00000100:00100000:0.0:1766100258.242909:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba0acca1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432979072:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100258.818956:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100259.838823:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100260.862317:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100261.888413:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100262.684883:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100262.684961:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100262.685009:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100262.685116:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249408
00000100:00100000:1.0:1766100262.685170:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1980
00000100:00100000:1.0:1766100262.685176:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249408:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100262.685252:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249408:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 75us (291us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100262.685260:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1980
00000100:00100000:3.0:1766100262.685459:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249536
00000100:00100000:3.0:1766100262.685525:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1833
00000100:00100000:3.0:1766100262.685529:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417249536:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100262.685622:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100262.685656:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417249536:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 126us (782us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100262.685663:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1833
00000100:00100000:2.0:1766100262.685992:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249472
00000100:00100000:2.0:1766100262.686033:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2927
00000100:00100000:2.0:1766100262.686039:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009470f4e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417249472:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100262.686099:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009470f4e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417249472:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 59us (1090us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100262.686104:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2927
00010000:00010000:0.0:1766100262.909802:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100263.363510:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100263.363517:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100263.363521:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100263.363529:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4766
00000100:00080000:3.0:1766100263.363532:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100263.363537:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100263.363556:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000048d06d88 x1851885432979520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100263.363934:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b836d1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432979648:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100263.364003:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100263.364037:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100263.364042:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003520602a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432979392:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100263.364068:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100263.364092:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002eba1753 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432979456:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100263.364116:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100263.364132:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100263.364137:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000763d20e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432979584:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100263.364157:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100263.364178:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048d06d88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432979520:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100263.364200:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100263.364600:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979648
00000100:00100000:0.0:1766100263.364637:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979520
00000100:00100000:0.0:1766100263.364655:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1254
00000100:00100000:0.0:1766100263.364661:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432979648:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100263.364714:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432979648:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (712us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100263.364721:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1254
00000100:00100000:0.0:1766100263.364741:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1834
00000100:00100000:0.0:1766100263.364746:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432979520:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:2.0:1766100263.364773:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979456
00000100:00100000:2.0:1766100263.364803:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979584
00000100:00100000:2.0:1766100263.364820:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2928
00000100:00100000:2.0:1766100263.364824:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c572263a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432979456:12345-0@lo:400:kworker/u8:2.0
00010000:02020000:0.0:1766100263.364835:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100263.364873:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c572263a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432979456:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (756us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100263.364880:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432979520:12345-0@lo:8:kworker/u8:2.0 Request processed in 132us (677us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100263.364896:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2928
00000100:00100000:0.0:1766100263.364904:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1834
00000100:00100000:0.0:1766100263.366689:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2929
00000100:00100000:0.0:1766100263.366702:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000411dc2e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432979584:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100263.366788:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000411dc2e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432979584:12345-0@lo:400:kworker/u8:2.0 Request processed in 89us (2630us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100263.366796:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2929
00000100:00100000:0.0:1766100263.366841:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979392
00000100:00100000:0.0:1766100263.366878:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1981
00000100:00100000:0.0:1766100263.366895:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432979392:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100263.366926:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432979392:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (2858us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100263.366932:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1981
00000100:00100000:0.0:1766100263.367028:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000763d20e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432979584:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100263.367041:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002eba1753 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432979456:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100263.367075:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003520602a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432979392:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766100263.367087:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002b836d1e x1851885432979648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100270 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766100263.367097:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b836d1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432979648:0@lo:400:kworker/u8:2.0
00000100:00080000:3.0:1766100263.367351:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100263.367357:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100263.367360:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048d06d88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432979520:0@lo:8:kworker/u8:2.0
00010000:00010000:1.0:1766100263.933657:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100264.957704:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100265.983173:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100267.006560:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100267.806586:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100267.806639:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100267.806721:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100267.806840:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249600
00000100:00100000:0.0:1766100267.806910:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1982
00000100:00100000:0.0:1766100267.806916:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417249600:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100267.806992:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417249600:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 74us (414us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100267.806999:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1982
00000100:00100000:3.0:1766100267.807152:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249728
00000100:00100000:3.0:1766100267.807185:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1835
00000100:00100000:3.0:1766100267.807189:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417249728:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100267.807288:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100267.807328:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417249728:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 137us (607us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100267.807336:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1835
00000100:00100000:2.0:1766100267.807523:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249664
00000100:00100000:2.0:1766100267.807569:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2930
00000100:00100000:2.0:1766100267.807574:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002288850c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417249664:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100267.807623:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002288850c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417249664:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 48us (985us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100267.807630:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2930
00010000:00010000:1.0:1766100268.030802:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100268.477874:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100268.477881:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100268.477884:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100268.477893:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4771
00000100:00080000:1.0:1766100268.477896:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100268.477902:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100268.477922:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a03d8323 x1851885432979840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100268.478002:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432979776:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478065:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100268.478109:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100268.478115:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432979904:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478137:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100268.478171:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979776
00000100:00100000:0.0:1766100268.478209:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979904
00000100:00100000:0.0:1766100268.478228:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2931
00000100:00100000:0.0:1766100268.478233:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001415cdc3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432979776:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478275:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001415cdc3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432979776:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (210us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100268.478281:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2931
00000100:00100000:0.0:1766100268.478296:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2932
00000100:00100000:0.0:1766100268.478299:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001047d83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432979904:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478318:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001047d83 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432979904:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (180us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100268.478322:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2932
00000100:00100000:0.0:1766100268.478338:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432979968:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478362:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100268.478378:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100268.478383:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432979712:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478405:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100268.478427:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979968
00000100:00100000:0.0:1766100268.478445:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1255
00000100:00100000:0.0:1766100268.478448:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432979968:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478473:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432979968:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (111us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100268.478497:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1255
00000100:00100000:0.0:1766100268.478513:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979712
00000100:00100000:0.0:1766100268.478530:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1983
00000100:00100000:0.0:1766100268.478533:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432979712:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478554:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432979712:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (149us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100268.478559:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1983
00000100:00100000:0.0:1766100268.478596:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ec69882 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432979712:0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766100268.478605:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b17b5997 x1851885432979968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100275 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766100268.478613:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b17b5997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432979968:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478634:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432979904:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100268.478640:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432979776:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100268.484611:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432979840:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100268.484683:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100268.484743:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432979840
00000100:00100000:3.0:1766100268.484772:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1836
00000100:00100000:3.0:1766100268.484777:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432979840:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766100268.484880:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100268.484931:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432979840:12345-0@lo:8:kworker/u8:2.0 Request processed in 153us (248us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100268.484938:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1836
00000100:00080000:0.0:1766100268.484993:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100268.484997:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100268.485001:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432979840:0@lo:8:kworker/u8:2.0
00010000:00010000:0.0:1766100269.055581:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100270.078012:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100271.102382:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100272.125687:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100272.924804:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100272.926176:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100272.926421:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249792
00000100:00100000:2.0:1766100272.926464:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1984
00000100:00100000:2.0:1766100272.926469:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417249792:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100272.926542:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417249792:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 72us (1747us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100272.926549:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1984
00000100:00100000:1.0:1766100272.931456:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249856
00000100:00100000:1.0:1766100272.931538:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2933
00000100:00100000:1.0:1766100272.931546:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000617de322 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417249856:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100272.931639:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000617de322 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417249856:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 89us (5463us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100272.931646:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2933
00000100:00100000:0.0:1766100272.932698:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100272.933177:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249920
00000100:00100000:3.0:1766100272.933219:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1837
00000100:00100000:3.0:1766100272.933225:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbb02080 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417249920:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100272.933316:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100272.933382:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbb02080 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417249920:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 156us (688us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100272.933389:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1837
00010000:00010000:1.0:1766100273.152234:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100273.600646:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce3f3b2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432980032:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100273.600721:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766100273.600953:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100273.600958:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100273.600961:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100273.600965:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4776
00000100:00080000:2.0:1766100273.600968:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100273.600973:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100273.600995:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000093f3d8a9 x1851885432980160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100273.601214:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980032
00000100:00100000:3.0:1766100273.601258:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1985
00000100:00100000:3.0:1766100273.601264:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432980032:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100273.602254:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432980288:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100273.602300:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100273.602554:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba0acca1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432980224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602586:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100273.602608:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100273.602614:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432980096:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602635:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100273.602673:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980224
00000100:00100000:2.0:1766100273.602705:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980096
00000100:00100000:2.0:1766100273.602722:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2934
00000100:00100000:2.0:1766100273.602726:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432980224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602767:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe6e47c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432980224:12345-0@lo:400:kworker/u8:3.0 Request processed in 39us (180us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100273.602774:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2934
00000100:00100000:2.0:1766100273.602791:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2935
00000100:00100000:2.0:1766100273.602794:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb80abc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432980096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602828:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb80abc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432980096:12345-0@lo:400:kworker/u8:3.0 Request processed in 33us (193us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100273.602834:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2935
00000100:00100000:2.0:1766100273.602871:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc68c7d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432980096:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602880:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba0acca1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432980224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602914:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093f3d8a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432980160:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100273.602941:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100273.602971:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980288
00000100:00100000:2.0:1766100273.602993:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980160
00000100:00100000:2.0:1766100273.603007:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1256
00000100:00100000:2.0:1766100273.603010:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432980288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100273.603038:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055153256 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432980288:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (739us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100273.603044:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1256
00000100:00100000:2.0:1766100273.603054:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1838
00000100:00100000:2.0:1766100273.603057:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba0acca1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432980160:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:2.0:1766100273.603151:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100273.603178:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba0acca1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432980160:12345-0@lo:8:kworker/u8:3.0 Request processed in 120us (237us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100273.603184:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1838
00000100:00080000:2.0:1766100273.603211:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100273.603214:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100273.603217:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093f3d8a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432980160:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100273.603400:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432980032:12345-0@lo:400:kworker/u8:3.0 Request processed in 2134us (2679us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100273.603408:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1985
00010000:00080000:0.0:1766100273.608523:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f12cf851 x1851885432980288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100280 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100273.608541:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432980288:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100273.610221:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce3f3b2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432980032:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100274.174332:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100275.197790:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100276.221702:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100277.247504:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100278.046044:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100278.046098:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100278.046182:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250048
00000100:00100000:2.0:1766100278.046241:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2936
00000100:00100000:2.0:1766100278.046247:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417250048:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100278.046276:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417249984
00000100:00100000:0.0:1766100278.046316:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1986
00000100:00100000:2.0:1766100278.046317:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000838122d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417250048:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 70us (219us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100278.046320:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249984:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100278.046324:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2936
00000100:00100000:0.0:1766100278.046346:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417249984:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 25us (311us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100278.046352:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1986
00000100:00100000:1.0:1766100278.047102:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100278.047371:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250112
00000100:00100000:2.0:1766100278.047409:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1839
00000100:00100000:2.0:1766100278.047413:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417250112:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100278.047529:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100278.047577:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417250112:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 163us (477us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100278.047584:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1839
00010000:00010000:0.0:1766100278.271685:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100278.718838:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100278.718845:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100278.718848:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100278.718856:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4782
00000100:00080000:1.0:1766100278.718858:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100278.718864:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100278.718894:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000a03d8323 x1851885432980480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100278.718937:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0c228e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432980416:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719000:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100278.719032:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100278.719039:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432980544:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719060:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100278.719083:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432980480:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719109:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100278.719143:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980416
00000100:00100000:1.0:1766100278.719182:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980544
00000100:00100000:1.0:1766100278.719205:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2937
00000100:00100000:1.0:1766100278.719211:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b17b5997 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432980416:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719255:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b17b5997 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432980416:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (254us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100278.719261:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2937
00000100:00100000:1.0:1766100278.719276:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2938
00000100:00100000:1.0:1766100278.719280:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432980544:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719305:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ec69882 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432980544:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (244us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100278.719310:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2938
00000100:00100000:1.0:1766100278.719328:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980480
00000100:00100000:1.0:1766100278.719347:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1840
00000100:00100000:1.0:1766100278.719351:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432980480:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:1.0:1766100278.719441:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100278.719472:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432980480:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (362us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100278.719507:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1840
00000100:00080000:1.0:1766100278.719544:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100278.719547:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100278.719551:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a03d8323 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432980480:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719579:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432980544:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.719588:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0c228e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432980416:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100278.719791:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432980352:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100278.719826:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100278.719858:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100278.719864:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432980608:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100278.719900:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100278.719926:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980608
00000100:00100000:3.0:1766100278.719952:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1257
00000100:00100000:3.0:1766100278.719955:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432980608:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100278.719989:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000116a4423 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432980608:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (89us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100278.719995:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1257
00010000:00080000:3.0:1766100278.720018:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000071bbd8cc x1851885432980608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100285 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766100278.720027:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432980608:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.725135:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980352
00000100:00100000:1.0:1766100278.725187:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1987
00000100:00100000:1.0:1766100278.725193:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432980352:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100278.725275:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432980352:12345-0@lo:400:kworker/u8:2.0 Request processed in 80us (5448us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100278.725282:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1987
00000100:00100000:1.0:1766100278.725350:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6e5460e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432980352:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100279.295211:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100280.323328:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100281.343625:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100282.365678:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100283.166722:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100283.166864:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250304
00000100:00100000:3.0:1766100283.166924:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1841
00000100:00100000:3.0:1766100283.166932:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417250304:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100283.167039:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100283.167623:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093f3d8a9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417250304:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 689us (907us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100283.167633:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1841
00000100:00100000:3.0:1766100283.168281:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100283.168405:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250176
00000100:00100000:3.0:1766100283.168419:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100283.168601:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1988
00000100:00100000:1.0:1766100283.168607:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250176:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100283.168669:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250176:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 61us (390us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100283.168676:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1988
00000100:00100000:2.0:1766100283.168771:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250240
00000100:00100000:2.0:1766100283.168809:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2939
00000100:00100000:2.0:1766100283.168813:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417250240:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100283.168857:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5012ca9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417250240:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 43us (438us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100283.168867:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2939
00010000:00010000:1.0:1766100283.390412:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100283.844890:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100283.844897:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100283.844900:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100283.844908:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4787
00000100:00080000:0.0:1766100283.844911:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100283.844916:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100283.844936:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e446c544 x1851885432980800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100283.845025:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2e7680e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432980928:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100283.845100:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100283.845138:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100283.845143:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e82385d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432980672:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100283.845169:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100283.845230:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980928
00000100:00100000:0.0:1766100283.845279:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1258
00000100:00100000:0.0:1766100283.845284:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc68c7d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432980928:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100283.845334:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc68c7d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432980928:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (236us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100283.845340:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1258
00010000:00080000:0.0:1766100283.845374:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d2e7680e x1851885432980928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100290 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766100283.845385:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2e7680e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432980928:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100283.845970:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001238be68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432980864:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100283.846007:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100283.847636:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e446c544 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432980800:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100283.847688:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100283.847762:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980800
00000100:00100000:0.0:1766100283.847793:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1842
00000100:00100000:0.0:1766100283.847798:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432980800:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766100283.847900:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100283.847936:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2e7680e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432980800:12345-0@lo:8:kworker/u8:2.0 Request processed in 137us (248us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100283.847943:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1842
00000100:00080000:0.0:1766100283.847978:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100283.847981:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100283.847985:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e446c544 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432980800:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100283.848012:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980672
00000100:00100000:0.0:1766100283.848037:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1989
00000100:00100000:0.0:1766100283.848041:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432980672:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100283.848076:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432980672:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (2907us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100283.848082:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1989
00000100:00100000:0.0:1766100283.848117:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e82385d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432980672:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100283.848192:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000987ca4ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432980736:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100283.850730:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980864
00000100:00100000:2.0:1766100283.850812:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2940
00000100:00100000:2.0:1766100283.850818:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432980864:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100283.850891:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6e5460e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432980864:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (4884us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100283.850898:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2940
00000100:00100000:0.0:1766100283.851036:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100283.851328:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980736
00000100:00100000:2.0:1766100283.851357:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2941
00000100:00100000:2.0:1766100283.851361:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e82385d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432980736:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100283.851392:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e82385d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432980736:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (357us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100283.851397:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2941
00000100:00100000:1.0:1766100283.851853:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001238be68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432980864:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100283.851932:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000987ca4ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432980736:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766100284.417570:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100285.440205:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100286.463636:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100287.486405:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100288.294071:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100288.294126:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100288.294183:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100288.294238:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250496
00000100:00100000:1.0:1766100288.294294:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1843
00000100:00100000:1.0:1766100288.294302:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417250496:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100288.294413:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100288.294528:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417250496:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 225us (345us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100288.294537:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1843
00000100:00100000:0.0:1766100288.294876:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250368
00000100:00100000:2.0:1766100288.294949:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250432
00000100:00100000:0.0:1766100288.294956:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1990
00000100:00100000:0.0:1766100288.294966:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250368:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100288.294999:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2942
00000100:00100000:2.0:1766100288.295006:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417250432:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100288.295045:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250368:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 77us (985us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100288.295055:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1990
00000100:00100000:2.0:1766100288.295062:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c228e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417250432:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 53us (933us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100288.295073:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2942
00010000:00010000:1.0:1766100288.509921:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100288.958805:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000987ca4ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432980992:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100288.958852:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e446c544 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432981056:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100288.959042:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100288.959084:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100288.959122:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432980992
00000100:00100000:3.0:1766100288.959179:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1991
00000100:00100000:3.0:1766100288.959187:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432980992:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100288.959241:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432980992:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (202us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100288.959248:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1991
00000100:00100000:3.0:1766100288.959305:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000987ca4ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432980992:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100288.959352:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981056
00000100:00100000:3.0:1766100288.959389:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2943
00000100:00100000:3.0:1766100288.959393:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff2504ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432981056:12345-0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766100288.960793:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100288.960802:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100288.960805:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100288.960812:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4792
00000100:00080000:0.0:1766100288.960816:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100288.960821:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100288.960857:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002d45f753 x1851885432981120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100288.960951:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d45f753 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432981120:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100288.961008:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100288.961048:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001238be68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432981248:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100288.961069:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100288.961092:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000987ca4ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432981184:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100288.961115:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100288.961148:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981120
00000100:00100000:0.0:1766100288.961175:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981248
00000100:00100000:0.0:1766100288.961198:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1259
00000100:00100000:0.0:1766100288.961202:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432981248:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100288.961281:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432981248:12345-0@lo:400:kworker/u8:2.0 Request processed in 77us (211us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100288.961293:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1259
00000100:00100000:0.0:1766100288.961319:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1844
00000100:00100000:0.0:1766100288.961323:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432981120:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100288.961396:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff2504ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432981056:12345-0@lo:400:kworker/u8:2.0 Request processed in 1996us (2312us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100288.961407:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2943
00010000:02020000:0.0:1766100288.961422:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100288.961435:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981184
00000100:00100000:0.0:1766100288.961467:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432981120:12345-0@lo:8:kworker/u8:2.0 Request processed in 142us (458us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100288.961497:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1844
00000100:00100000:3.0:1766100288.961545:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2944
00000100:00100000:3.0:1766100288.961551:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aea833b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432981184:12345-0@lo:400:kworker/u8:2.0
00010000:00080000:0.0:1766100288.961557:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000001238be68 x1851885432981248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100295 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:2.0:1766100288.961565:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e446c544 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432981056:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100288.961571:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001238be68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432981248:0@lo:400:kworker/u8:2.0
00000100:00080000:0.0:1766100288.961610:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00100000:3.0:1766100288.961617:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aea833b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432981184:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (497us total) trans 0 rc 0/0
00000100:00080000:0.0:1766100288.961619:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100288.961627:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d45f753 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432981120:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100288.961636:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2944
00000100:00100000:0.0:1766100288.961673:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000987ca4ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432981184:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100289.537924:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100290.557741:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100291.581743:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100292.605915:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100293.407334:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100293.408305:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250560
00000100:00100000:0.0:1766100293.408383:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1992
00000100:00100000:0.0:1766100293.408392:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417250560:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100293.408528:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417250560:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 137us (1206us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100293.408536:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1992
00000100:00100000:1.0:1766100293.420972:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100293.422554:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100293.422659:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250688
00000100:00100000:0.0:1766100293.422726:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1845
00000100:00100000:0.0:1766100293.422734:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071bbd8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417250688:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100293.422832:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100293.423181:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071bbd8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417250688:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 446us (635us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100293.423190:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1845
00000100:00100000:3.0:1766100293.423316:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250624
00000100:00100000:3.0:1766100293.423364:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2945
00000100:00100000:3.0:1766100293.423369:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a03d8323 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417250624:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100293.423415:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a03d8323 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417250624:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 46us (2455us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100293.423422:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2945
00010000:00010000:0.0:1766100293.631704:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100294.079946:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100294.079956:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100294.079961:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100294.079971:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4797
00000100:00080000:2.0:1766100294.079975:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100294.079983:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100294.080018:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000020a14350 x1851885432981440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100294.080123:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432981376:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.080227:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100294.080271:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100294.080282:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432981504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.080305:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100294.080343:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981376
00000100:00100000:3.0:1766100294.080393:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981504
00000100:00100000:3.0:1766100294.080414:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2946
00000100:00100000:3.0:1766100294.080420:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432981376:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.080520:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048d06d88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432981376:12345-0@lo:400:kworker/u8:3.0 Request processed in 99us (302us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100294.080528:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2946
00000100:00100000:3.0:1766100294.080549:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2947
00000100:00100000:3.0:1766100294.080552:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432981504:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.080582:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9759d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432981504:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (276us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100294.080588:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2947
00000100:00100000:3.0:1766100294.080641:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432981504:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.080653:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f12cf851 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432981376:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100294.080789:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432981568:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100294.080828:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100294.080859:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766100294.080865:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce3f3b2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432981312:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100294.080893:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100294.080969:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981568
00000100:00100000:3.0:1766100294.081010:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1260
00000100:00100000:3.0:1766100294.081014:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de7892db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432981568:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.081052:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de7892db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432981568:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (224us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100294.081058:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1260
00000100:00100000:3.0:1766100294.081077:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981312
00000100:00100000:3.0:1766100294.081099:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1993
00000100:00100000:3.0:1766100294.081103:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432981312:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100294.081136:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432981312:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (243us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100294.081142:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1993
00000100:00100000:3.0:1766100294.081173:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce3f3b2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432981312:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766100294.081184:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005e10322b x1851885432981568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100301 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100294.081193:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432981568:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100294.081524:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020a14350 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432981440:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100294.081561:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100294.081669:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981440
00000100:00100000:1.0:1766100294.081702:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1846
00000100:00100000:1.0:1766100294.081708:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432981440:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100294.081808:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100294.081857:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432981440:12345-0@lo:8:kworker/u8:3.0 Request processed in 147us (295us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100294.081864:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1846
00000100:00080000:0.0:1766100294.081968:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100294.081973:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100294.081982:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020a14350 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432981440:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766100294.653763:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100295.678683:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100296.701768:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100297.725852:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100298.525920:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100298.532599:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100298.532715:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100298.532804:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250816
00000100:00100000:2.0:1766100298.532882:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2948
00000100:00100000:2.0:1766100298.532890:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d1df7f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417250816:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100298.532997:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d1df7f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417250816:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 107us (411us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100298.533006:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2948
00000100:00100000:3.0:1766100298.533291:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250880
00000100:00100000:3.0:1766100298.533326:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1847
00000100:00100000:3.0:1766100298.533331:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417250880:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100298.533437:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100298.533510:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417250880:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 177us (795us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100298.533517:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1847
00000100:00100000:0.0:1766100298.534749:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250752
00000100:00100000:0.0:1766100298.534841:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1994
00000100:00100000:0.0:1766100298.534847:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250752:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100298.534921:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250752:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 74us (9013us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100298.534930:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1994
00010000:00010000:1.0:1766100298.749810:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100299.198545:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100299.198553:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100299.198557:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100299.198567:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4802
00000100:00080000:2.0:1766100299.198572:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100299.198579:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100299.198605:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000437b852e x1851885432981760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100299.198744:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432981888:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.198832:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100299.198869:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100299.198875:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432981632:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.198905:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100299.198931:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432981696:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.198970:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100299.198989:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100299.198994:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432981824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199012:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100299.199045:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981888
00000100:00100000:3.0:1766100299.199088:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1261
00000100:00100000:3.0:1766100299.199095:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432981888:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199143:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b836d1e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432981888:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (312us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100299.199150:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1261
00000100:00100000:3.0:1766100299.199172:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981632
00000100:00100000:3.0:1766100299.199190:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1995
00000100:00100000:3.0:1766100299.199193:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432981632:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199214:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432981632:12345-0@lo:400:kworker/u8:3.0 Request processed in 21us (310us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100299.199219:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1995
00000100:00100000:3.0:1766100299.199243:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981696
00000100:00100000:3.0:1766100299.199262:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981824
00000100:00100000:3.0:1766100299.199289:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2949
00000100:00100000:3.0:1766100299.199292:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003520602a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432981696:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199319:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003520602a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432981696:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (349us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100299.199324:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2949
00000100:00100000:3.0:1766100299.199332:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2950
00000100:00100000:3.0:1766100299.199334:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432981824:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199350:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eba1753 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432981824:12345-0@lo:400:kworker/u8:3.0 Request processed in 15us (338us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100299.199358:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2950
00000100:00100000:3.0:1766100299.199401:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432981824:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199408:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432981696:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100299.199429:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432981632:0@lo:400:kworker/u8:3.0
00010000:00080000:3.0:1766100299.199437:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008bdf5e63 x1851885432981888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100306 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100299.199444:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432981888:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100299.199571:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432981760:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100299.199600:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100299.199690:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981760
00000100:00100000:3.0:1766100299.199715:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1848
00000100:00100000:3.0:1766100299.199718:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432981760:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100299.199815:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100299.199850:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432981760:12345-0@lo:8:kworker/u8:3.0 Request processed in 130us (249us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100299.199856:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1848
00000100:00080000:0.0:1766100299.199918:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100299.199923:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100299.199927:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432981760:0@lo:8:kworker/u8:3.0
00010000:00010000:0.0:1766100299.773811:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100300.798751:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100301.823208:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100302.845811:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100303.646797:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100303.646943:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417250944
00000100:00100000:0.0:1766100303.647014:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1996
00000100:00100000:0.0:1766100303.647022:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250944:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100303.647120:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417250944:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 98us (335us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100303.647129:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1996
00000100:00100000:1.0:1766100303.648931:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100303.649104:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100303.649220:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251008
00000100:00100000:3.0:1766100303.649280:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2951
00000100:00100000:3.0:1766100303.649286:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5f4f40e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251008:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100303.649366:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5f4f40e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251008:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 80us (445us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100303.649374:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2951
00000100:00100000:0.0:1766100303.652193:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251072
00000100:00100000:0.0:1766100303.652269:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1849
00000100:00100000:0.0:1766100303.652277:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157816be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417251072:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100303.652394:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100303.652504:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157816be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417251072:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 226us (3400us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100303.652513:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1849
00010000:00010000:1.0:1766100303.872899:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100304.320141:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100304.320151:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100304.320155:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100304.320166:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4807
00000100:00080000:2.0:1766100304.320171:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100304.320178:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100304.320207:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008bdf5e63 x1851885432982080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100304.320656:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432982144:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.320744:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100304.320798:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100304.320803:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432982016:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.320822:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100304.320850:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432982080:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100304.320877:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100304.320919:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982144
00000100:00100000:0.0:1766100304.320967:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982016
00000100:00100000:0.0:1766100304.320986:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2952
00000100:00100000:0.0:1766100304.320992:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000da5bbb9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432982144:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321050:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000da5bbb9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432982144:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (306us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100304.321058:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2952
00000100:00100000:0.0:1766100304.321081:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2953
00000100:00100000:0.0:1766100304.321085:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432982016:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321110:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e446c544 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432982016:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (287us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100304.321116:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2953
00000100:00100000:0.0:1766100304.321136:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982080
00000100:00100000:0.0:1766100304.321154:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1850
00000100:00100000:0.0:1766100304.321158:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432982080:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100304.321271:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100304.321306:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000987ca4ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432982080:12345-0@lo:8:kworker/u8:3.0 Request processed in 147us (429us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100304.321312:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1850
00000100:00100000:0.0:1766100304.321350:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432981952:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321383:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100304.321403:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100304.321408:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432982208:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321429:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100304.321450:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432981952
00000100:00100000:0.0:1766100304.321528:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1997
00000100:00100000:0.0:1766100304.321532:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432981952:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321561:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432981952:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (179us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100304.321568:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1997
00000100:00100000:0.0:1766100304.321615:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982208
00000100:00100000:0.0:1766100304.321637:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1262
00000100:00100000:0.0:1766100304.321641:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432982208:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321668:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d45f753 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432982208:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (239us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100304.321674:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1262
00000100:00080000:0.0:1766100304.321728:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100304.321733:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100304.321737:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432982080:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766100304.321769:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b6bb6a94 x1851885432982208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100311 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100304.321782:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432982208:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321789:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432981952:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321815:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000437b852e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432982016:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100304.321825:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432982144:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100304.893729:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100305.918861:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100306.941728:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100307.970829:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100308.768346:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100308.768415:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100308.768842:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251264
00000100:00100000:0.0:1766100308.769953:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1851
00000100:00100000:0.0:1766100308.769965:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251264:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100308.770895:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100308.771683:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251136
00000100:00100000:2.0:1766100308.771763:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1998
00000100:00100000:2.0:1766100308.771770:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417251136:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100308.773566:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251264:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 3580us (5212us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100308.773589:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1851
00000100:00100000:1.0:1766100308.774271:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100308.774973:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251200
00000100:00100000:2.0:1766100308.775196:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417251136:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 3421us (6777us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100308.775212:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1998
00000100:00100000:0.0:1766100308.776087:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2954
00000100:00100000:0.0:1766100308.776099:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052df8ad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417251200:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100308.776199:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052df8ad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417251200:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 101us (1933us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100308.776207:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2954
00010000:00010000:0.0:1766100308.990509:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100309.438032:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766100309.439093:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100309.439100:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100309.439105:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100309.439116:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4812
00000100:00080000:2.0:1766100309.439121:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100309.439128:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100309.439158:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000005e10322b x1851885432982400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100309.439698:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432982272:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.439796:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100309.439885:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982272
00000100:00100000:3.0:1766100309.439938:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1999
00000100:00100000:3.0:1766100309.439945:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432982272:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.440001:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432982272:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (213us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100309.440009:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1999
00000100:00100000:3.0:1766100309.440072:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432982272:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.440276:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432982528:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.440318:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100309.440929:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020a14350 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432982336:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100309.440978:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100309.441014:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100309.441021:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432982464:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100309.441047:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100309.441083:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432982400:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100309.441111:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100309.441147:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982528
00000100:00100000:2.0:1766100309.441261:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982336
00000100:00100000:2.0:1766100309.441624:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982464
00000100:00100000:3.0:1766100309.441679:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2955
00000100:00100000:3.0:1766100309.441685:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432982336:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.441752:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001238be68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432982336:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (774us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100309.441761:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2955
00000100:00100000:3.0:1766100309.441780:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2956
00000100:00100000:3.0:1766100309.441783:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432982464:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.441809:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a55df54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432982464:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (762us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100309.441815:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2956
00000100:00100000:3.0:1766100309.441853:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432982464:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.441865:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020a14350 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432982336:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100309.441894:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982400
00000100:00100000:0.0:1766100309.442688:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1263
00000100:00100000:0.0:1766100309.442696:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432982528:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100309.442759:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000763d20e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432982528:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (2441us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100309.442765:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1263
00000100:00100000:0.0:1766100309.442784:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1852
00000100:00100000:0.0:1766100309.442788:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432982400:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:0.0:1766100309.442905:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100309.442944:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcd4c4c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432982400:12345-0@lo:8:kworker/u8:2.0 Request processed in 154us (1832us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100309.442950:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1852
00000100:00080000:1.0:1766100309.443143:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100309.443149:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100309.443153:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432982400:0@lo:8:kworker/u8:2.0
00010000:00080000:1.0:1766100309.443223:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000009b57ab98 x1851885432982528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100316 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:1.0:1766100309.443234:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432982528:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100310.013736:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100311.038132:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100312.066542:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100313.085843:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100313.886956:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100313.888965:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251328
00000100:00100000:3.0:1766100313.889052:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2000
00000100:00100000:3.0:1766100313.889060:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417251328:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100313.889167:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417251328:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 107us (2224us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100313.889175:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2000
00000100:00100000:2.0:1766100313.890270:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100313.891526:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100313.893775:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251392
00000100:00100000:1.0:1766100313.893871:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2957
00000100:00100000:1.0:1766100313.893880:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000437b852e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417251392:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100313.894006:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000437b852e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417251392:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 127us (2498us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100313.894015:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2957
00000100:00100000:0.0:1766100313.895140:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251456
00000100:00100000:0.0:1766100313.895203:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1853
00000100:00100000:0.0:1766100313.895210:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251456:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100313.895320:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100313.913041:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251456:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 17822us (22768us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100313.913059:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1853
00010000:00010000:1.0:1766100314.111617:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100314.557668:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100314.557676:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100314.557680:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100314.557690:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4817
00000100:00080000:2.0:1766100314.557693:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100314.557701:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100314.557726:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008bdf5e63 x1851885432982720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100314.557839:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432982848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100314.557921:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100314.557970:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100314.557974:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432982592:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100314.558003:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100314.558040:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982592
00000100:00100000:3.0:1766100314.558083:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2001
00000100:00100000:3.0:1766100314.558089:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432982592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100314.558136:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432982592:12345-0@lo:400:kworker/u8:3.0 Request processed in 48us (133us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100314.558144:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2001
00000100:00100000:3.0:1766100314.558187:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432982592:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100314.558257:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982848
00000100:00100000:2.0:1766100314.558292:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1264
00000100:00100000:2.0:1766100314.558295:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce3f3b2c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432982848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100314.558332:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce3f3b2c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432982848:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (412us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100314.558338:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1264
00000100:00100000:0.0:1766100314.558436:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432982656:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100314.558469:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100314.558530:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100314.558535:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432982784:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100314.558554:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100314.558630:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432982720:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100314.558670:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100314.558757:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982656
00000100:00100000:2.0:1766100314.558783:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982784
00000100:00100000:3.0:1766100314.558830:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982720
00000100:00100000:3.0:1766100314.558847:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1854
00000100:00100000:3.0:1766100314.558851:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000742bc8a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432982720:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100314.558948:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100314.558999:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000742bc8a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432982720:12345-0@lo:8:kworker/u8:3.0 Request processed in 147us (330us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100314.559005:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1854
00000100:00080000:3.0:1766100314.559051:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100314.559054:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100314.559059:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432982720:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766100314.559082:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000005e10322b x1851885432982848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100321 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100314.559093:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e10322b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432982848:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100314.559167:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2958
00000100:00100000:2.0:1766100314.559171:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432982656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100314.559208:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a16d6f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432982656:12345-0@lo:400:kworker/u8:3.0 Request processed in 36us (739us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100314.559213:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2958
00000100:00100000:2.0:1766100314.559223:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2959
00000100:00100000:2.0:1766100314.559226:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432982784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100314.559245:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055c81506 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432982784:12345-0@lo:400:kworker/u8:3.0 Request processed in 18us (690us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100314.559251:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2959
00000100:00100000:0.0:1766100314.561847:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b57ab98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432982784:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100314.561871:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432982656:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100315.135047:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100316.157866:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100317.182557:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100318.205961:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100319.005789:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100319.005852:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100319.005878:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251584
00000100:00100000:0.0:1766100319.005958:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2960
00000100:00100000:0.0:1766100319.005967:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020a14350 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251584:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100319.006065:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020a14350 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251584:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 97us (299us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100319.006072:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2960
00000100:00100000:3.0:1766100319.006280:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251520
00000100:00100000:3.0:1766100319.006325:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2002
00000100:00100000:3.0:1766100319.006332:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417251520:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100319.006369:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d4f5ba2d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417251520:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 37us (517us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100319.006378:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2002
00000100:00100000:2.0:1766100319.006959:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100319.013759:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251648
00000100:00100000:3.0:1766100319.014008:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1855
00000100:00100000:3.0:1766100319.014018:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6e93a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251648:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100319.014143:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100319.014255:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6e93a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251648:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 237us (7298us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100319.014264:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1855
00010000:00010000:1.0:1766100319.229809:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100319.678159:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6a8540f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432982912:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.678260:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766100319.678337:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100319.678344:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100319.678349:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100319.678354:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4823
00000100:00080000:0.0:1766100319.678358:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100319.678366:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100319.678391:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000094cabff9 x1851885432983040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100319.678448:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100319.678456:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093648709 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432983168:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.678526:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100319.678575:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eb9fd8f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432982976:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.678607:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100319.678630:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100319.678635:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432983104:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.678661:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100319.679078:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094cabff9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432983040:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100319.679117:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100319.679159:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983168
00000100:00100000:3.0:1766100319.679222:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983040
00000100:00100000:3.0:1766100319.679246:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1265
00000100:00100000:3.0:1766100319.679253:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432983168:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100319.679312:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432983168:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (787us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100319.679321:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1265
00000100:00100000:3.0:1766100319.679341:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1856
00000100:00100000:3.0:1766100319.679344:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432983040:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100319.679447:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100319.679696:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982976
00000100:00100000:2.0:1766100319.680116:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983104
00000100:00100000:0.0:1766100319.680845:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432982912
00000100:00100000:0.0:1766100319.680889:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2003
00000100:00100000:0.0:1766100319.680894:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432982912:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100319.680985:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000583c8c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432982912:12345-0@lo:400:kworker/u8:3.0 Request processed in 90us (2727us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100319.680993:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2003
00000100:00100000:3.0:1766100319.681307:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f12cf851 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432983040:12345-0@lo:8:kworker/u8:3.0 Request processed in 1959us (2188us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100319.681316:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1856
00000100:00100000:2.0:1766100319.681428:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2961
00000100:00100000:2.0:1766100319.681434:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b57ab98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432982976:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.681511:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b57ab98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432982976:12345-0@lo:400:kworker/u8:3.0 Request processed in 76us (2903us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100319.681517:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2961
00000100:00100000:2.0:1766100319.681531:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2962
00000100:00100000:2.0:1766100319.681536:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e10322b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432983104:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.681563:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e10322b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432983104:12345-0@lo:400:kworker/u8:3.0 Request processed in 27us (2903us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100319.681569:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2962
00000100:00100000:2.0:1766100319.681631:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432983104:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.681647:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eb9fd8f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432982976:0@lo:400:kworker/u8:3.0
00010000:00080000:2.0:1766100319.681709:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000093648709 x1851885432983168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100326 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100319.681720:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093648709 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432983168:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100319.681730:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6a8540f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432982912:0@lo:400:kworker/u8:3.0
00000100:00080000:2.0:1766100319.681781:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100319.681787:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100319.681791:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094cabff9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432983040:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766100320.253906:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100321.277830:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100322.303712:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100323.329436:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100324.125589:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100324.125656:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100324.125800:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251776
00000100:00100000:2.0:1766100324.125889:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2963
00000100:00100000:2.0:1766100324.125899:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dad21f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251776:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100324.126003:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dad21f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251776:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 105us (426us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100324.126013:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2963
00000100:00100000:0.0:1766100324.126434:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100324.126578:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251712
00000100:00100000:3.0:1766100324.126626:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2004
00000100:00100000:3.0:1766100324.126632:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417251712:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100324.126690:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b547ee6a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417251712:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 58us (1035us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100324.126698:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2004
00000100:00100000:2.0:1766100324.127086:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251840
00000100:00100000:2.0:1766100324.128104:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1857
00000100:00100000:2.0:1766100324.128112:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251840:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100324.128220:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100324.128297:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417251840:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 184us (1864us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100324.128304:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1857
00010000:00010000:0.0:1766100324.351367:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100324.797855:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100324.797866:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100324.797870:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100324.797880:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4828
00000100:00080000:1.0:1766100324.797884:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100324.797891:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100324.797902:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766100324.797924:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000009f4bdd76 x1851885432983360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100324.797929:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc29c393 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432983232:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798004:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1ae343a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432983296:0@lo:400:kworker/u8:2.0
00000100:00100000:2.0:1766100324.798028:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100324.798132:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983232
00000100:00100000:0.0:1766100324.798186:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2005
00000100:00100000:0.0:1766100324.798194:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432983232:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100324.798266:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100324.798275:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea384f3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432983488:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798281:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100324.798311:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100324.798317:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432983424:0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100324.798328:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100324.798347:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100324.798373:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983488
00000100:00100000:1.0:1766100324.798385:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f4bdd76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432983360:0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798411:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100324.798414:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1266
00000100:00100000:3.0:1766100324.798420:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f180bc23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432983488:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798453:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983296
00000100:00100000:1.0:1766100324.798509:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983424
00000100:00100000:3.0:1766100324.798512:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f180bc23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432983488:12345-0@lo:400:kworker/u8:2.0 Request processed in 89us (184us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100324.798522:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1266
00000100:00100000:1.0:1766100324.798531:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2964
00000100:00100000:1.0:1766100324.798537:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432983296:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100324.798548:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983360
00000100:00100000:1.0:1766100324.798577:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017b40edd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432983296:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (299us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100324.798579:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1858
00000100:00100000:3.0:1766100324.798585:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432983360:12345-0@lo:8:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798589:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2964
00000100:00100000:1.0:1766100324.798606:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2965
00000100:00100000:1.0:1766100324.798609:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432983424:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798629:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dda6ad2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432983424:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (284us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100324.798634:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2965
00000100:00100000:1.0:1766100324.798671:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065874e88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432983424:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100324.798680:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1ae343a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432983296:0@lo:400:kworker/u8:2.0
00010000:02020000:3.0:1766100324.798706:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100324.798749:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432983360:12345-0@lo:8:kworker/u8:2.0 Request processed in 163us (339us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100324.798756:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1858
00000100:00080000:3.0:1766100324.798797:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100324.798802:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100324.798806:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f4bdd76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432983360:0@lo:8:kworker/u8:2.0
00010000:00080000:3.0:1766100324.798837:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000ea384f3a x1851885432983488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100331 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:3.0:1766100324.798845:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea384f3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432983488:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100324.800008:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa43e208 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432983232:12345-0@lo:400:kworker/u8:2.0 Request processed in 1808us (1977us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100324.800020:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2005
00000100:00100000:3.0:1766100324.800257:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc29c393 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432983232:0@lo:400:kworker/u8:2.0
00010000:00010000:1.0:1766100325.373917:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100326.398327:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100327.423617:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100328.446677:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100329.245668:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100329.245722:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100329.246987:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251904
00000100:00100000:2.0:1766100329.247082:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2006
00000100:00100000:2.0:1766100329.247092:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417251904:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100329.247209:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000036b760f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417251904:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 117us (1551us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100329.247217:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2006
00000100:00100000:0.0:1766100329.247791:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100329.249215:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417251968
00000100:00100000:1.0:1766100329.249374:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2966
00000100:00100000:1.0:1766100329.249382:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000669f53de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251968:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100329.249525:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000669f53de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417251968:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 143us (3802us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100329.249534:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2966
00000100:00100000:0.0:1766100329.251735:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252032
00000100:00100000:0.0:1766100329.251808:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1859
00000100:00100000:0.0:1766100329.251816:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252032:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100329.251918:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100329.252017:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252032:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 201us (4229us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100329.252025:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1859
00010000:00010000:1.0:1766100329.469859:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100329.921315:0:13190:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100329.921324:0:13190:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100329.921328:0:13190:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100329.921338:0:13190:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4833
00000100:00080000:3.0:1766100329.921341:0:13190:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100329.921348:0:13190:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100329.921371:0:13190:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000e9626520 x1851885432983680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100329.921428:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9626520 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432983680:0@lo:8:kworker/u8:2.0
00000100:00100000:3.0:1766100329.921533:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100329.921588:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983680
00000100:00100000:3.0:1766100329.921623:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1860
00000100:00100000:3.0:1766100329.921629:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432983680:12345-0@lo:8:kworker/u8:2.0
00010000:02020000:3.0:1766100329.921729:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100329.921776:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432983680:12345-0@lo:8:kworker/u8:2.0 Request processed in 147us (244us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100329.921784:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1860
00000100:00080000:3.0:1766100329.921825:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100329.921828:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100329.921833:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9626520 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432983680:0@lo:8:kworker/u8:2.0
00000100:00100000:0.0:1766100329.921916:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000052d7190 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432983552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100329.921950:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100329.921978:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100329.921983:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432983808:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100329.922004:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100329.922025:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983808
00000100:00100000:0.0:1766100329.922052:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1267
00000100:00100000:0.0:1766100329.922055:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f484c134 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432983808:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100329.922090:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f484c134 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432983808:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (87us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100329.922096:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1267
00010000:00080000:0.0:1766100329.922124:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000032a0d422 x1851885432983808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100336 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:2.0'
00000100:00100000:0.0:1766100329.922135:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432983808:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100329.923206:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033ac3a12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432983616:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100329.923267:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100329.923309:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100329.923315:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065002652 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432983744:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100329.923338:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100329.923561:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983552
00000100:00100000:3.0:1766100329.923607:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2007
00000100:00100000:3.0:1766100329.923612:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432983552:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:3.0:1766100329.923673:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000339a4eaa pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432983552:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (1722us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100329.923680:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2007
00000100:00100000:3.0:1766100329.923736:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000052d7190 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432983552:0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100329.924299:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983616
00000100:00100000:0.0:1766100329.924334:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983744
00000100:00100000:0.0:1766100329.924350:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2967
00000100:00100000:0.0:1766100329.924353:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1ae343a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432983616:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100329.924391:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1ae343a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432983616:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (1126us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100329.924396:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2967
00000100:00100000:0.0:1766100329.924418:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2968
00000100:00100000:0.0:1766100329.924421:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065874e88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432983744:12345-0@lo:400:kworker/u8:2.0
00000100:00100000:0.0:1766100329.924440:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065874e88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432983744:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (1102us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100329.924445:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2968
00000100:00100000:1.0:1766100329.924524:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065002652 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432983744:0@lo:400:kworker/u8:2.0
00000100:00100000:1.0:1766100329.924536:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033ac3a12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432983616:0@lo:400:kworker/u8:2.0
00010000:00010000:0.0:1766100330.499304:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100331.517827:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100332.541715:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100333.567182:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100334.366006:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100334.366089:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100334.366301:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252160
00000100:00100000:2.0:1766100334.366387:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2969
00000100:00100000:2.0:1766100334.366395:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000052d7190 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417252160:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100334.366532:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000052d7190 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417252160:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 137us (443us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100334.366541:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2969
00000100:00100000:0.0:1766100334.366628:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252096
00000100:00100000:0.0:1766100334.366670:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2008
00000100:00100000:0.0:1766100334.366675:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417252096:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100334.366705:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043efe98b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417252096:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 29us (712us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100334.366711:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2008
00000100:00100000:3.0:1766100334.374729:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100334.374848:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252224
00000100:00100000:1.0:1766100334.374900:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1861
00000100:00100000:1.0:1766100334.374907:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252224:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100334.375006:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100334.375095:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252224:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 187us (380us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100334.375103:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1861
00010000:00010000:1.0:1766100334.589775:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100335.039276:0:15193:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100335.039285:0:15193:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100335.039289:0:15193:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100335.039299:0:15193:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4838
00000100:00080000:3.0:1766100335.039303:0:15193:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100335.039311:0:15193:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100335.039337:0:15193:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000006f2f7702 x1851885432984000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100335.039401:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000532dc8bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432983872:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.039655:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432983936:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100335.039670:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432984128:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.039692:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100335.039708:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100335.039724:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100335.039728:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432984064:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100335.039747:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984128
00000100:00100000:0.0:1766100335.039749:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100335.039780:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983936
00000100:00100000:2.0:1766100335.039795:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1268
00000100:00100000:2.0:1766100335.039802:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efc5e5a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432984128:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.039804:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984064
00000100:00100000:0.0:1766100335.039819:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2970
00000100:00100000:0.0:1766100335.039823:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000caf250b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432983936:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.039856:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000caf250b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432983936:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (164us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100335.039856:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efc5e5a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432984128:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (147us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100335.039864:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2970
00000100:00100000:2.0:1766100335.039864:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1268
00000100:00100000:0.0:1766100335.039884:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2971
00000100:00100000:0.0:1766100335.039886:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d287ef1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432984064:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.039907:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d287ef1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432984064:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (159us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100335.039913:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2971
00000100:00100000:0.0:1766100335.039950:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432984064:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.039960:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432983936:0@lo:400:kworker/u8:0.0
00010000:00080000:2.0:1766100335.040022:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000006cd3ba3 x1851885432984128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100342 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:0.0'
00000100:00100000:2.0:1766100335.040037:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432984128:0@lo:400:kworker/u8:0.0
00000100:00100000:2.0:1766100335.040058:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984000:0@lo:8:kworker/u8:0.0
00000100:00100000:2.0:1766100335.040084:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100335.040769:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100335.044118:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432983872
00000100:00100000:1.0:1766100335.044187:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2009
00000100:00100000:1.0:1766100335.044194:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432983872:12345-0@lo:400:kworker/u8:0.0
00000100:00100000:1.0:1766100335.044294:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024aa41d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432983872:12345-0@lo:400:kworker/u8:0.0 Request processed in 100us (4798us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100335.044302:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2009
00000100:00100000:1.0:1766100335.044393:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000532dc8bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432983872:0@lo:400:kworker/u8:0.0
00000100:00100000:0.0:1766100335.045615:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984000
00000100:00100000:0.0:1766100335.045677:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1862
00000100:00100000:0.0:1766100335.045683:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432984000:12345-0@lo:8:kworker/u8:0.0
00010000:02020000:0.0:1766100335.045788:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100335.045877:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432984000:12345-0@lo:8:kworker/u8:0.0 Request processed in 193us (5792us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100335.045885:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1862
00000100:00080000:2.0:1766100335.046032:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100335.046038:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100335.046043:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984000:0@lo:8:kworker/u8:0.0
00010000:00010000:1.0:1766100335.626272:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100336.637665:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100337.661864:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100338.687223:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100339.484638:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100339.484694:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100339.484803:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252352
00000100:00100000:0.0:1766100339.484867:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2972
00000100:00100000:0.0:1766100339.484874:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417252352:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100339.484960:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019b6a62d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417252352:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 86us (333us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100339.484968:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2972
00000100:00100000:2.0:1766100339.485385:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252288
00000100:00100000:2.0:1766100339.485422:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2010
00000100:00100000:2.0:1766100339.485427:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417252288:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100339.485462:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f61b962 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417252288:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 35us (769us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100339.485468:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2010
00000100:00100000:1.0:1766100339.487532:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100339.487650:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252416
00000100:00100000:0.0:1766100339.487708:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1863
00000100:00100000:0.0:1766100339.487724:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252416:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100339.487831:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100339.487908:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252416:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 184us (386us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100339.487916:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1863
00010000:00010000:0.0:1766100339.710202:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100340.160097:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:2.0:1766100340.160104:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100340.160110:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100340.160114:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:0.0:1766100340.160119:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432984192:0@lo:400:kworker/u8:1.0
00000100:00080000:2.0:1766100340.160121:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4843
00000100:00080000:2.0:1766100340.160124:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100340.160131:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100340.160153:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008bdf5e63 x1851885432984320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100340.160198:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984320:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100340.160202:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100340.160233:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432984448:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100340.160233:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100340.160253:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100340.160275:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432984384:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100340.160279:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984320
00000100:00100000:0.0:1766100340.160316:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100340.160342:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:2.0:1766100340.160342:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984448
00000100:00100000:0.0:1766100340.160347:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432984256:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100340.160367:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100340.160374:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1269
00000100:00100000:2.0:1766100340.160381:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7f92201 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432984448:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100340.160393:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1864
00000100:00100000:0.0:1766100340.160396:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432984320:12345-0@lo:8:kworker/u8:1.0
00000100:00100000:2.0:1766100340.160436:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7f92201 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432984448:12345-0@lo:400:kworker/u8:1.0 Request processed in 54us (182us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100340.160443:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1269
00010000:02020000:0.0:1766100340.160511:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100340.160551:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432984320:12345-0@lo:8:kworker/u8:1.0 Request processed in 153us (318us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100340.160557:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1864
00000100:00100000:0.0:1766100340.160579:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984192
00000100:00100000:0.0:1766100340.160603:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2011
00000100:00100000:0.0:1766100340.160607:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432984192:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100340.160608:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984384
00000100:00100000:2.0:1766100340.160636:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984256
00000100:00100000:0.0:1766100340.160637:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ca8c665 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432984192:12345-0@lo:400:kworker/u8:1.0 Request processed in 30us (436us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100340.160643:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2011
00000100:00100000:2.0:1766100340.160654:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2973
00000100:00100000:2.0:1766100340.160657:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe958a22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432984384:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100340.160678:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2974
00000100:00100000:0.0:1766100340.160682:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077a1e307 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432984256:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100340.160689:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe958a22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432984384:12345-0@lo:400:kworker/u8:1.0 Request processed in 31us (374us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100340.160696:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2973
00000100:00100000:0.0:1766100340.160704:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077a1e307 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432984256:12345-0@lo:400:kworker/u8:1.0 Request processed in 22us (337us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100340.160710:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2974
00000100:00080000:0.0:1766100340.160742:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100340.160746:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100340.160750:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984320:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100340.160783:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432984256:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100340.160793:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045df1f7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432984384:0@lo:400:kworker/u8:1.0
00010000:00080000:1.0:1766100340.160806:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000addcb14d x1851885432984448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100347 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:1.0:1766100340.160822:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432984448:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100340.160832:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432984192:0@lo:400:kworker/u8:1.0
00010000:00010000:1.0:1766100340.734525:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100341.758514:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100342.781773:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100343.805772:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100344.604648:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100344.604777:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252480
00000100:00100000:0.0:1766100344.604856:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2012
00000100:00100000:0.0:1766100344.604865:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417252480:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100344.604967:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddc4e0b0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417252480:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 102us (332us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100344.604976:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2012
00000100:00100000:3.0:1766100344.610590:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100344.610700:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252608
00000100:00100000:2.0:1766100344.611775:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1865
00000100:00100000:2.0:1766100344.611786:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252608:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100344.611894:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100344.612008:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252608:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 221us (1428us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100344.612017:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1865
00000100:00100000:1.0:1766100344.614328:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100344.614455:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252544
00000100:00100000:2.0:1766100344.614556:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2975
00000100:00100000:2.0:1766100344.614565:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417252544:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100344.614682:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ea27af9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417252544:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 116us (365us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100344.614692:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2975
00010000:00010000:2.0:1766100344.834952:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100345.280701:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de7892db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432984512:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100345.280911:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:1.0:1766100345.281508:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100345.281515:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100345.281520:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100345.281527:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4848
00000100:00080000:1.0:1766100345.281532:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100345.281540:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100345.281570:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000157816be x1851885432984640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100345.281985:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000742bc8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432984768:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282195:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100345.282249:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a733f90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432984704:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282280:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100345.282313:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100345.282320:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094cd88d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432984576:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282344:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100345.282372:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000157816be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984640:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282399:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100345.282442:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984768
00000100:00100000:0.0:1766100345.282528:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984640
00000100:00100000:0.0:1766100345.282553:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1270
00000100:00100000:0.0:1766100345.282561:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c41f2c96 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432984768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282621:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c41f2c96 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432984768:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (428us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100345.282629:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1270
00000100:00100000:0.0:1766100345.282657:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1866
00000100:00100000:0.0:1766100345.282661:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad90c4a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432984640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100345.282776:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100345.282811:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad90c4a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432984640:12345-0@lo:8:kworker/u8:3.0 Request processed in 149us (412us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100345.282818:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1866
00000100:00100000:0.0:1766100345.282839:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984704
00000100:00100000:0.0:1766100345.282867:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984576
00000100:00100000:0.0:1766100345.282888:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2976
00000100:00100000:0.0:1766100345.282892:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014e0e83d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432984704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282927:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014e0e83d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432984704:12345-0@lo:400:kworker/u8:3.0 Request processed in 35us (648us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100345.282934:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2976
00000100:00100000:0.0:1766100345.282947:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2977
00000100:00100000:0.0:1766100345.282950:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000030a9820 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432984576:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.282976:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000030a9820 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432984576:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (633us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100345.282983:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2977
00000100:00080000:0.0:1766100345.283035:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100345.283040:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100345.283044:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000157816be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984640:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100345.283080:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094cd88d9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432984576:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.283091:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a733f90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432984704:0@lo:400:kworker/u8:3.0
00010000:00080000:0.0:1766100345.283118:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000742bc8a8 x1851885432984768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100352 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100345.283128:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000742bc8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432984768:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.283189:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984512
00000100:00100000:0.0:1766100345.283217:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013
00000100:00100000:0.0:1766100345.283222:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432984512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100345.290314:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000477df5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432984512:12345-0@lo:400:kworker/u8:3.0 Request processed in 7077us (9392us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100345.290336:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013
00000100:00100000:3.0:1766100345.290994:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de7892db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432984512:0@lo:400:kworker/u8:3.0
00010000:00010000:0.0:1766100345.856313:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100346.890750:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100347.903901:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:0.0:1766100348.926513:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100349.724803:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100349.724870:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100349.724993:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252800
00000100:00100000:0.0:1766100349.725059:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1867
00000100:00100000:0.0:1766100349.725067:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252800:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:0.0:1766100349.725173:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100349.725290:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417252800:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 221us (496us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100349.725298:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1867
00000100:00100000:1.0:1766100349.725628:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252672
00000100:00100000:1.0:1766100349.725674:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2014
00000100:00100000:1.0:1766100349.725679:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417252672:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100349.725735:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057425092 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417252672:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 54us (865us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100349.725742:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2014
00000100:00100000:3.0:1766100349.738006:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100349.738175:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252736
00000100:00100000:2.0:1766100349.738274:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2978
00000100:00100000:2.0:1766100349.738282:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417252736:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100349.738394:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020ca4bf7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417252736:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 112us (401us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100349.738403:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2978
00010000:00010000:1.0:1766100349.949843:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100350.397801:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100350.397810:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100350.397815:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100350.397825:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4853
00000100:00080000:0.0:1766100350.397829:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100350.397836:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100350.397861:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000000f28b7aa x1851885432984960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100350.397919:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432985024:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.397998:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100350.398039:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100350.398045:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ef33bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432984896:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.398064:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100350.398087:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984960:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100350.398115:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100350.398168:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985024
00000100:00100000:2.0:1766100350.398217:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000585c6915 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432984832:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.398222:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984896
00000100:00100000:0.0:1766100350.398268:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2979
00000100:00100000:0.0:1766100350.398276:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000ae936e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432985024:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100350.398306:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100350.398438:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432985088:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100350.398567:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984960
00000100:00100000:3.0:1766100350.398594:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1868
00000100:00100000:3.0:1766100350.398599:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432984960:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:3.0:1766100350.398699:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100350.403128:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000ae936e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432985024:12345-0@lo:400:kworker/u8:1.0 Request processed in 4840us (5119us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100350.403150:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2979
00000100:00100000:0.0:1766100350.403205:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2980
00000100:00100000:0.0:1766100350.403213:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde0997c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432984896:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.403307:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde0997c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432984896:12345-0@lo:400:kworker/u8:1.0 Request processed in 95us (5242us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100350.403313:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2980
00000100:00100000:0.0:1766100350.403384:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ef33bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432984896:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.403399:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432985024:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.403470:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432984832
00000100:00100000:0.0:1766100350.403528:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2015
00000100:00100000:0.0:1766100350.403532:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432984832:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.403568:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddcbc21 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432984832:12345-0@lo:400:kworker/u8:1.0 Request processed in 34us (5263us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100350.403574:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2015
00000100:00100000:2.0:1766100350.404419:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000585c6915 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432984832:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100350.404944:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100350.405278:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985088
00000100:00100000:0.0:1766100350.405341:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1271
00000100:00100000:0.0:1766100350.405348:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432985088:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100350.405413:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432985088:12345-0@lo:400:kworker/u8:1.0 Request processed in 65us (473us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100350.405420:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1271
00010000:00080000:1.0:1766100350.405449:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000c402e8c6 x1851885432985088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100357 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:1.0:1766100350.405463:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c402e8c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432985088:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100350.406928:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432984960:12345-0@lo:8:kworker/u8:1.0 Request processed in 8316us (8800us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100350.406947:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1868
00000100:00080000:0.0:1766100350.407297:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100350.407302:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100350.407307:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432984960:0@lo:8:kworker/u8:1.0
00010000:00010000:2.0:1766100350.978732:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100351.998537:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100353.024199:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100354.046448:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100354.845190:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100354.845335:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252864
00000100:00100000:0.0:1766100354.845408:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2016
00000100:00100000:0.0:1766100354.845416:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417252864:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100354.845544:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091259148 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417252864:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 127us (365us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100354.845554:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2016
00000100:00100000:1.0:1766100354.846527:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100354.846871:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252928
00000100:00100000:1.0:1766100354.847289:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100354.847349:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417252992
00000100:00100000:1.0:1766100354.847381:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1869
00000100:00100000:1.0:1766100354.847388:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417252992:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100354.847525:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100354.847589:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417252992:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 202us (304us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100354.847602:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1869
00000100:00100000:0.0:1766100354.847862:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2981
00000100:00100000:0.0:1766100354.847868:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc29c393 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417252928:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100354.849892:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc29c393 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417252928:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 2012us (3357us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100354.849958:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2981
00010000:00010000:2.0:1766100355.069787:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100355.517770:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100355.517779:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100355.517784:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100355.517794:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4858
00000100:00080000:2.0:1766100355.517799:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100355.517806:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100355.517816:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432985216:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100355.517830:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000addcb14d x1851885432985280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100355.517904:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100355.517930:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432985408:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100355.517953:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100355.517959:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432985344:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100355.517967:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100355.517978:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100355.517988:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100355.517993:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432985152:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100355.518013:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985408
00000100:00100000:0.0:1766100355.518032:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100355.518062:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432985280:0@lo:8:kworker/u8:1.0
00000100:00100000:1.0:1766100355.518063:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1272
00000100:00100000:1.0:1766100355.518070:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f28b7aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432985408:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100355.518085:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100355.518113:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985280
00000100:00100000:1.0:1766100355.518123:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f28b7aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432985408:12345-0@lo:400:kworker/u8:1.0 Request processed in 52us (155us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100355.518130:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1272
00000100:00100000:0.0:1766100355.518135:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1870
00000100:00100000:0.0:1766100355.518138:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e96bbcf1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432985280:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:0.0:1766100355.518243:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100355.518278:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e96bbcf1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432985280:12345-0@lo:8:kworker/u8:1.0 Request processed in 139us (193us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100355.518285:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1870
00000100:00080000:0.0:1766100355.518328:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100355.518332:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100355.518336:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432985280:0@lo:8:kworker/u8:1.0
00010000:00080000:0.0:1766100355.518373:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000706cdd21 x1851885432985408/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100362 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:0.0:1766100355.518383:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432985408:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100355.518909:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985216
00000100:00100000:2.0:1766100355.518994:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985344
00000100:00100000:0.0:1766100355.519153:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2982
00000100:00100000:0.0:1766100355.519158:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f4bdd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432985216:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100355.519261:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f4bdd76 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432985216:12345-0@lo:400:kworker/u8:1.0 Request processed in 102us (1364us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100355.519268:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2982
00000100:00100000:0.0:1766100355.519285:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2983
00000100:00100000:0.0:1766100355.519288:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de7892db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432985344:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100355.519315:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de7892db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432985344:12345-0@lo:400:kworker/u8:1.0 Request processed in 26us (1336us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100355.519321:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2983
00000100:00100000:2.0:1766100355.519360:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985152
00000100:00100000:1.0:1766100355.519375:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432985344:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100355.519387:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2017
00000100:00100000:1.0:1766100355.519389:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432985216:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100355.519391:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432985152:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100355.519429:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000038be66 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432985152:12345-0@lo:400:kworker/u8:1.0 Request processed in 37us (1397us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100355.519435:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2017
00000100:00100000:2.0:1766100355.519513:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432985152:0@lo:400:kworker/u8:1.0
00010000:00010000:1.0:1766100356.093984:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100357.120454:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100358.141981:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100359.166413:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100359.965913:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100359.965969:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100359.966065:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100359.966196:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253056
00000100:00100000:2.0:1766100359.966266:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2018
00000100:00100000:2.0:1766100359.966275:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417253056:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100359.966365:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a5b383f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417253056:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 90us (464us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100359.966373:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2018
00000100:00100000:0.0:1766100359.966645:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253120
00000100:00100000:0.0:1766100359.966692:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2984
00000100:00100000:0.0:1766100359.966697:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000742bc8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417253120:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100359.966740:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000742bc8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417253120:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 41us (771us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100359.966746:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2984
00000100:00100000:3.0:1766100359.967027:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253184
00000100:00100000:3.0:1766100359.967071:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1871
00000100:00100000:3.0:1766100359.967076:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417253184:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100359.967172:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100359.967217:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417253184:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 140us (1153us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100359.967224:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1871
00010000:00010000:2.0:1766100360.190624:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100360.638331:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100360.638341:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100360.638346:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100360.638356:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4863
00000100:00080000:3.0:1766100360.638360:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100360.638366:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100360.638394:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000006cd3ba3 x1851885432985600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100360.638460:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432985728:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.638567:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100360.638621:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100360.638627:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432985472:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.638678:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100360.638715:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000532dc8bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432985536:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.638746:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100360.638766:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100360.638771:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432985664:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.638794:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100360.638844:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985536
00000100:00100000:3.0:1766100360.638895:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985664
00000100:00100000:3.0:1766100360.638919:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2985
00000100:00100000:3.0:1766100360.638925:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432985536:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.638986:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088aa54c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432985536:12345-0@lo:400:kworker/u8:3.0 Request processed in 60us (239us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100360.638994:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2985
00000100:00100000:3.0:1766100360.639012:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2986
00000100:00100000:3.0:1766100360.639016:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432985664:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.639041:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9626520 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432985664:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (248us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100360.639047:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2986
00000100:00100000:3.0:1766100360.639083:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432985664:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.639093:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000532dc8bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432985536:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.639137:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985472
00000100:00100000:3.0:1766100360.639161:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2019
00000100:00100000:3.0:1766100360.639164:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432985472:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100360.639212:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005038bdfc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432985472:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (534us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100360.639219:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2019
00000100:00100000:3.0:1766100360.639250:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432985472:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100360.639361:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432985600:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100360.639393:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100360.639437:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985728
00000100:00100000:0.0:1766100360.639465:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985600
00000100:00100000:0.0:1766100360.639504:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1273
00000100:00100000:0.0:1766100360.639508:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f180bc23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432985728:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100360.639543:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f180bc23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432985728:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (979us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100360.639549:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1273
00000100:00100000:0.0:1766100360.639563:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1872
00000100:00100000:0.0:1766100360.639566:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef33bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432985600:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100360.639662:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100360.639694:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef33bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432985600:12345-0@lo:8:kworker/u8:3.0 Request processed in 126us (300us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100360.639700:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1872
00000100:00080000:0.0:1766100360.639742:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100360.639746:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100360.639749:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006cd3ba3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432985600:0@lo:8:kworker/u8:3.0
00010000:00080000:0.0:1766100360.639775:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002c75ca47 x1851885432985728/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100367 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100360.639785:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432985728:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100361.214600:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100362.291510:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100363.261864:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100364.285662:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100365.087716:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100365.087785:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100365.088452:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253248
00000100:00100000:2.0:1766100365.088562:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2020
00000100:00100000:2.0:1766100365.088571:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417253248:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100365.088674:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a1bb7a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417253248:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 103us (888us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100365.088683:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2020
00000100:00100000:0.0:1766100365.089066:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253376
00000100:00100000:0.0:1766100365.089107:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1873
00000100:00100000:0.0:1766100365.089112:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006f2f7702 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417253376:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100365.089231:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100365.089703:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006f2f7702 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417253376:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 588us (2006us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100365.089713:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1873
00000100:00100000:3.0:1766100365.094057:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100365.094182:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253312
00000100:00100000:0.0:1766100365.094265:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2987
00000100:00100000:0.0:1766100365.094274:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000532dc8bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417253312:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100365.094379:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000532dc8bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417253312:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 106us (334us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100365.094388:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2987
00010000:00010000:1.0:1766100365.309758:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100365.763590:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1766100365.763598:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:3.0:1766100365.763600:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100365.763605:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100365.763616:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4869
00000100:00080000:3.0:1766100365.763622:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00100000:2.0:1766100365.763624:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432985792:0@lo:400:kworker/u8:1.0
00000100:00080000:3.0:1766100365.763631:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100365.763658:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000065002652 x1851885432985920/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100365.763708:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033ac3a12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432985856:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100365.763716:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100365.763753:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100365.763756:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432986048:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100365.763780:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100365.763787:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432985984:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100365.763792:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100365.763901:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986048
00000100:00100000:0.0:1766100365.763953:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1274
00000100:00100000:0.0:1766100365.763960:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432986048:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100365.764956:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100365.764982:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985792
00000100:00100000:3.0:1766100365.765006:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985856
00000100:00100000:1.0:1766100365.765028:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2021
00000100:00100000:1.0:1766100365.765033:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432985792:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100365.765047:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985984
00000100:00100000:3.0:1766100365.765069:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2988
00000100:00100000:3.0:1766100365.765074:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006cd3ba3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432985856:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100365.765085:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f84762d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432985792:12345-0@lo:400:kworker/u8:1.0 Request processed in 50us (1371us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100365.765094:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2021
00000100:00100000:3.0:1766100365.765116:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006cd3ba3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432985856:12345-0@lo:400:kworker/u8:1.0 Request processed in 41us (1364us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100365.765123:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2988
00000100:00100000:1.0:1766100365.765135:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065002652 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432985920:0@lo:8:kworker/u8:1.0
00000100:00100000:3.0:1766100365.765149:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2989
00000100:00100000:3.0:1766100365.765154:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043ef6b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432985984:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100365.765186:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100365.765189:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043ef6b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432985984:12345-0@lo:400:kworker/u8:1.0 Request processed in 32us (238us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100365.765195:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2989
00000100:00100000:1.0:1766100365.765242:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432985920
00000100:00100000:3.0:1766100365.765251:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432985984:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100365.765266:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1874
00000100:00100000:3.0:1766100365.765267:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033ac3a12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432985856:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100365.765272:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432985920:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:1.0:1766100365.765378:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100365.765410:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432985920:12345-0@lo:8:kworker/u8:1.0 Request processed in 138us (227us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100365.765416:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1874
00000100:00080000:1.0:1766100365.765440:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100365.765444:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100365.765447:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065002652 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432985920:0@lo:8:kworker/u8:1.0
00000100:00100000:2.0:1766100365.766170:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c57c659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432985792:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100365.767657:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432986048:12345-0@lo:400:kworker/u8:1.0 Request processed in 3687us (3858us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100365.767675:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1274
00010000:00080000:2.0:1766100365.767800:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002c75ca47 x1851885432986048/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100372 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:2.0:1766100365.767816:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432986048:0@lo:400:kworker/u8:1.0
00010000:00010000:2.0:1766100366.333755:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100367.357666:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100368.381789:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100369.406107:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100370.210735:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100370.210799:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100370.210848:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100370.210952:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253504
00000100:00100000:0.0:1766100370.211027:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2990
00000100:00100000:0.0:1766100370.211036:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8d3efcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417253504:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100370.211142:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8d3efcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417253504:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 106us (343us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100370.211154:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2990
00000100:00100000:2.0:1766100370.211234:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253568
00000100:00100000:2.0:1766100370.211269:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1875
00000100:00100000:2.0:1766100370.211274:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047cad513 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417253568:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100370.211372:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100370.211407:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047cad513 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417253568:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 132us (560us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100370.211413:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1875
00000100:00100000:1.0:1766100370.211543:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253440
00000100:00100000:1.0:1766100370.211581:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2022
00000100:00100000:1.0:1766100370.211585:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417253440:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100370.211746:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815246c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417253440:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 159us (1156us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100370.211753:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2022
00010000:00010000:2.0:1766100370.429726:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1766100370.878035:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100370.878044:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100370.878048:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100370.878058:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4874
00000100:00080000:0.0:1766100370.878061:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100370.878068:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100370.878090:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000094cabff9 x1851885432986240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100370.878647:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432986176:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100370.878735:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100370.878777:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100370.878783:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6a8540f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432986304:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100370.878802:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100370.878837:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986176
00000100:00100000:3.0:1766100370.878884:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986304
00000100:00100000:3.0:1766100370.878902:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2991
00000100:00100000:3.0:1766100370.878908:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c51052e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432986176:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100370.878957:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c51052e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432986176:12345-0@lo:400:kworker/u8:1.0 Request processed in 48us (222us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100370.878964:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2991
00000100:00100000:3.0:1766100370.878981:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2992
00000100:00100000:3.0:1766100370.878985:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070167dff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432986304:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100370.879007:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070167dff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432986304:12345-0@lo:400:kworker/u8:1.0 Request processed in 22us (205us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100370.879012:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2992
00000100:00100000:3.0:1766100370.879056:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6a8540f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432986304:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100370.879065:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432986176:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100370.879256:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093648709 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432986368:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100370.879295:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100370.879324:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:2.0:1766100370.879330:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432986112:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100370.879357:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100370.879437:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986112
00000100:00100000:3.0:1766100370.879496:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2023
00000100:00100000:3.0:1766100370.879501:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432986112:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100370.879536:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004170a6cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432986112:12345-0@lo:400:kworker/u8:1.0 Request processed in 34us (179us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100370.879546:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2023
00000100:00100000:3.0:1766100370.879588:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432986112:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100370.879712:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986368
00000100:00100000:0.0:1766100370.879740:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1275
00000100:00100000:0.0:1766100370.879744:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432986368:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100370.879782:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432986368:12345-0@lo:400:kworker/u8:1.0 Request processed in 37us (487us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100370.879788:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1275
00010000:00080000:3.0:1766100370.879852:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000093648709 x1851885432986368/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100377 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:3.0:1766100370.879864:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093648709 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432986368:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100370.881441:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094cabff9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432986240:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100370.881540:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100370.881751:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986240
00000100:00100000:2.0:1766100370.881788:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1876
00000100:00100000:2.0:1766100370.881794:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eb9fd8f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432986240:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:2.0:1766100370.881900:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100370.881949:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eb9fd8f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432986240:12345-0@lo:8:kworker/u8:1.0 Request processed in 153us (410us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100370.881956:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1876
00000100:00080000:0.0:1766100370.882115:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100370.882121:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100370.882125:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094cabff9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432986240:0@lo:8:kworker/u8:1.0
00010000:00010000:1.0:1766100371.455608:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100372.477751:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100373.501982:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100374.525787:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100375.324582:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100375.324657:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100375.324658:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253696
00000100:00100000:0.0:1766100375.324736:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2993
00000100:00100000:0.0:1766100375.324746:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157816be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417253696:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100375.324764:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253632
00000100:00100000:0.0:1766100375.324850:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157816be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417253696:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 104us (280us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100375.324859:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2993
00000100:00100000:2.0:1766100375.325547:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2024
00000100:00100000:2.0:1766100375.325554:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417253632:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100375.325608:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff5b27a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417253632:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 54us (951us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100375.325615:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2024
00000100:00100000:1.0:1766100375.327383:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100375.330209:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253760
00000100:00100000:2.0:1766100375.330298:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1877
00000100:00100000:2.0:1766100375.330306:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417253760:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100375.330412:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100375.330550:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417253760:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 243us (3176us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100375.330559:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1877
00010000:00010000:2.0:1766100375.549774:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100376.004259:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100376.004268:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:1.0:1766100376.004273:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432986432:0@lo:400:kworker/u8:1.0
00000100:00080000:3.0:1766100376.004273:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100376.004280:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4879
00000100:00080000:3.0:1766100376.004283:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100376.004289:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100376.004313:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000008d577608 x1851885432986560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100376.004365:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432986496:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100376.004368:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100376.004401:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100376.004411:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:1.0:1766100376.004415:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432986688:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100376.004421:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100376.004425:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432986624:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100376.004441:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100376.004443:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100376.004508:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986432
00000100:00100000:3.0:1766100376.004514:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986496
00000100:00100000:3.0:1766100376.004554:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986624
00000100:00100000:1.0:1766100376.004558:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025
00000100:00100000:1.0:1766100376.004565:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432986432:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100376.004571:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2994
00000100:00100000:3.0:1766100376.004574:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c57c659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432986496:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100376.004612:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aab9aac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432986432:12345-0@lo:400:kworker/u8:1.0 Request processed in 47us (246us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100376.004612:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c57c659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432986496:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (210us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100376.004620:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025
00000100:00100000:3.0:1766100376.004622:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2994
00000100:00100000:3.0:1766100376.004636:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2995
00000100:00100000:3.0:1766100376.004639:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065002652 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432986624:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100376.004658:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986688
00000100:00100000:3.0:1766100376.004675:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065002652 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432986624:12345-0@lo:400:kworker/u8:1.0 Request processed in 35us (231us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100376.004682:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1276
00000100:00100000:1.0:1766100376.004686:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000458dfff5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432986688:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100376.004714:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000458dfff5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432986688:12345-0@lo:400:kworker/u8:1.0 Request processed in 27us (272us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100376.004719:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1276
00010000:00080000:1.0:1766100376.004756:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002c75ca47 x1851885432986688/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100383 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:1.0:1766100376.004770:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432986688:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100376.004780:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432986432:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100376.004966:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432986560:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100376.004999:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100376.005070:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432986624:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100376.005082:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7b9f638 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432986496:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100376.005280:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2995
00000100:00100000:1.0:1766100376.006747:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986560
00000100:00100000:1.0:1766100376.006796:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1878
00000100:00100000:1.0:1766100376.006803:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cabff9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432986560:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:1.0:1766100376.006903:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00080000:3.0:1766100376.010117:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100376.010131:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100376.010142:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432986560:0@lo:8:kworker/u8:1.0
00000100:00100000:1.0:1766100376.010593:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cabff9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432986560:12345-0@lo:8:kworker/u8:1.0 Request processed in 3786us (5590us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100376.010607:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1878
00010000:00010000:2.0:1766100376.576156:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100377.598582:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100378.624156:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100379.647294:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100380.446887:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100380.448543:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100380.449164:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253888
00000100:00100000:0.0:1766100380.449262:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2996
00000100:00100000:0.0:1766100380.449271:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ac3a12 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417253888:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100380.450257:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100380.451576:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ac3a12 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417253888:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 2294us (4691us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100380.451595:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2996
00000100:00100000:1.0:1766100380.452086:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253824
00000100:00100000:1.0:1766100380.452162:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2026
00000100:00100000:1.0:1766100380.452169:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417253824:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100380.452256:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ae4446f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417253824:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 86us (3723us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100380.452263:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2026
00000100:00100000:2.0:1766100380.453152:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417253952
00000100:00100000:2.0:1766100380.453213:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1879
00000100:00100000:2.0:1766100380.453219:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000032a0d422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417253952:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100380.453328:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100380.453404:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000032a0d422 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417253952:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 184us (3151us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100380.453412:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1879
00010000:00010000:1.0:1766100380.669902:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100381.119227:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432986752:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100381.119326:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100381.119430:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986752
00000100:00100000:3.0:1766100381.119522:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2027
00000100:00100000:3.0:1766100381.119529:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432986752:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100381.120362:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99dd1ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432986752:12345-0@lo:400:kworker/u8:1.0 Request processed in 827us (1031us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100381.120377:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2027
00000100:00100000:1.0:1766100381.120581:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432986752:0@lo:400:kworker/u8:1.0
00000100:00080000:0.0:1766100381.124055:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100381.124065:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00100000:1.0:1766100381.124068:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:0.0:1766100381.124070:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100381.124082:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4884
00000100:00080000:0.0:1766100381.124086:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100381.124094:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100381.124097:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432986816:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100381.124124:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000093648709 x1851885432986880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100381.124192:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100381.124201:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100381.124212:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432986944:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100381.124245:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100381.124249:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100381.124258:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432987008:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100381.124296:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986816
00000100:00100000:1.0:1766100381.124297:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100381.124333:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987008
00000100:00100000:0.0:1766100381.124348:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986944
00000100:00100000:1.0:1766100381.124375:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1277
00000100:00100000:0.0:1766100381.124379:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2997
00000100:00100000:1.0:1766100381.124384:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c5bf9cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432987008:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100381.124388:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5354685 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432986816:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100381.124452:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c5bf9cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432987008:12345-0@lo:400:kworker/u8:1.0 Request processed in 69us (156us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100381.124458:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5354685 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432986816:12345-0@lo:400:kworker/u8:1.0 Request processed in 72us (268us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100381.124463:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1277
00000100:00100000:0.0:1766100381.124471:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2997
00000100:00100000:0.0:1766100381.124517:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2998
00000100:00100000:0.0:1766100381.124522:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6a8540f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432986944:12345-0@lo:400:kworker/u8:1.0
00010000:00080000:1.0:1766100381.124546:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f484c134 x1851885432987008/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100388 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:0.0:1766100381.124555:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6a8540f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432986944:12345-0@lo:400:kworker/u8:1.0 Request processed in 32us (308us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100381.124562:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432987008:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100381.124578:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2998
00000100:00100000:2.0:1766100381.124960:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093648709 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432986880:0@lo:8:kworker/u8:1.0
00000100:00100000:2.0:1766100381.125017:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100381.125049:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432986944:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100381.125117:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432986816:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100381.125181:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432986880
00000100:00100000:1.0:1766100381.125220:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1880
00000100:00100000:1.0:1766100381.125225:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432986880:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:1.0:1766100381.125324:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100381.125377:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432986880:12345-0@lo:8:kworker/u8:1.0 Request processed in 151us (360us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100381.125384:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1880
00000100:00080000:0.0:1766100381.125536:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100381.125541:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100381.125545:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093648709 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432986880:0@lo:8:kworker/u8:1.0
00010000:00010000:2.0:1766100381.693880:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100382.717938:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100383.744160:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100384.766963:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100385.569072:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100385.569148:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100385.569261:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254080
00000100:00100000:2.0:1766100385.569350:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2999
00000100:00100000:2.0:1766100385.569358:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093648709 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417254080:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100385.569463:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093648709 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417254080:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 105us (403us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100385.569472:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2999
00000100:00100000:1.0:1766100385.569793:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254016
00000100:00100000:1.0:1766100385.569834:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2028
00000100:00100000:1.0:1766100385.569839:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417254016:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100385.569887:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd44c726 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417254016:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 47us (739us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100385.569894:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2028
00000100:00100000:0.0:1766100385.578957:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100385.579128:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254144
00000100:00100000:2.0:1766100385.579193:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1881
00000100:00100000:2.0:1766100385.579201:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d8990d8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417254144:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100385.579316:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100385.579407:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d8990d8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417254144:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 205us (461us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100385.579416:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1881
00010000:00010000:2.0:1766100385.791760:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100386.241253:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100386.241264:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100386.241268:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100386.241280:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4889
00000100:00080000:3.0:1766100386.241291:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100386.241300:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100386.241333:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000004833a3bc x1851885432987200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100386.241399:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003fede54a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432987136:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100386.241511:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100386.241551:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100386.241558:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000965f5dd0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432987264:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100386.241596:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100386.241629:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004833a3bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432987200:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100386.241660:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100386.241702:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987136
00000100:00100000:3.0:1766100386.241749:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987264
00000100:00100000:3.0:1766100386.241772:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3000
00000100:00100000:3.0:1766100386.241779:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000098532044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432987136:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100386.241839:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000098532044 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432987136:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (329us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100386.241848:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3000
00000100:00100000:3.0:1766100386.241874:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3001
00000100:00100000:3.0:1766100386.241878:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cb6e8c2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432987264:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100386.241927:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003fede54a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432987136:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100386.242142:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987200
00000100:00100000:0.0:1766100386.242143:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6d2bcd4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432987328:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100386.242176:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1882
00000100:00100000:1.0:1766100386.242181:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000449764d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432987200:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100386.242292:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100386.243043:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432987072:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100386.243592:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000449764d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432987200:12345-0@lo:8:kworker/u8:3.0 Request processed in 1407us (1929us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100386.243604:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1882
00000100:00080000:1.0:1766100386.243684:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100386.243689:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100386.243694:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004833a3bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432987200:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100386.243741:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100386.243829:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987072
00000100:00100000:3.0:1766100386.243944:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cb6e8c2c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432987264:12345-0@lo:400:kworker/u8:3.0 Request processed in 2063us (2346us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100386.243953:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3001
00000100:00100000:1.0:1766100386.244686:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000965f5dd0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432987264:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100386.244688:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100386.244784:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987328
00000100:00100000:3.0:1766100386.244830:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1278
00000100:00100000:3.0:1766100386.244843:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432987328:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100386.244909:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432987328:12345-0@lo:400:kworker/u8:3.0 Request processed in 66us (223us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100386.244916:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1278
00010000:00080000:0.0:1766100386.245252:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000d6d2bcd4 x1851885432987328/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100393 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100386.245265:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6d2bcd4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432987328:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100386.245374:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2029
00000100:00100000:2.0:1766100386.245379:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432987072:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100386.245438:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000158fc4e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432987072:12345-0@lo:400:kworker/u8:3.0 Request processed in 58us (1699us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100386.245444:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2029
00000100:00100000:3.0:1766100386.245957:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432987072:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100386.814936:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100387.842634:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100388.862652:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100389.885784:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100390.687859:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100390.687910:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100390.687963:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100390.688071:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254336
00000100:00100000:1.0:1766100390.688134:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1883
00000100:00100000:1.0:1766100390.688142:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000585c6915 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417254336:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100390.688248:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100390.688329:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000585c6915 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417254336:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 187us (418us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100390.688337:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1883
00000100:00100000:2.0:1766100390.689085:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254272
00000100:00100000:2.0:1766100390.689144:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3002
00000100:00100000:2.0:1766100390.689149:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c402e8c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417254272:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100390.689206:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c402e8c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417254272:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 56us (1359us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100390.689212:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3002
00000100:00100000:3.0:1766100390.689662:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254208
00000100:00100000:3.0:1766100390.689706:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2030
00000100:00100000:3.0:1766100390.689710:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417254208:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100390.689754:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096fe326e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417254208:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 43us (1799us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100390.689761:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2030
00010000:00010000:2.0:1766100390.909853:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100391.362369:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432987392:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100391.362445:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100391.362554:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987392
00000100:00100000:1.0:1766100391.362601:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2031
00000100:00100000:1.0:1766100391.362607:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432987392:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100391.363588:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100391.363594:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100391.363597:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100391.363601:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4894
00000100:00080000:0.0:1766100391.363604:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100391.363608:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100391.363626:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000071bbd8cc x1851885432987520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100391.363728:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000734b74d6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432987392:12345-0@lo:400:kworker/u8:3.0 Request processed in 1120us (1284us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100391.363737:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2031
00000100:00100000:1.0:1766100391.363776:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432987456:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.363809:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100391.363830:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100391.363835:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432987584:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.363857:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100391.363885:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987456
00000100:00100000:1.0:1766100391.363912:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987584
00000100:00100000:1.0:1766100391.363933:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3003
00000100:00100000:1.0:1766100391.363937:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d75985ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432987456:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.363987:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d75985ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432987456:12345-0@lo:400:kworker/u8:3.0 Request processed in 49us (178us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100391.363993:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3003
00000100:00100000:1.0:1766100391.364004:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3004
00000100:00100000:1.0:1766100391.364007:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c1ee83e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432987584:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.364032:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c1ee83e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432987584:12345-0@lo:400:kworker/u8:3.0 Request processed in 24us (175us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100391.364037:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3004
00000100:00100000:1.0:1766100391.364078:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432987584:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.364089:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432987456:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.364121:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432987520:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100391.364148:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100391.364183:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432987648:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.364205:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100391.364231:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987520
00000100:00100000:1.0:1766100391.364249:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987648
00000100:00100000:1.0:1766100391.364267:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1279
00000100:00100000:1.0:1766100391.364270:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432987648:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100391.364303:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432987648:12345-0@lo:400:kworker/u8:3.0 Request processed in 32us (98us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100391.364309:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1279
00000100:00100000:1.0:1766100391.364319:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884
00000100:00100000:1.0:1766100391.364322:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432987520:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100391.364411:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100391.364441:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432987520:12345-0@lo:8:kworker/u8:3.0 Request processed in 117us (292us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100391.364446:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884
00010000:00080000:1.0:1766100391.364468:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000039f6e93a x1851885432987648/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100398 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100391.364496:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432987648:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100391.364518:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100391.364521:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100391.364525:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071bbd8cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432987520:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100391.365177:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f484c134 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432987392:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100391.934332:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100392.958270:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100393.982537:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100395.008159:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100395.804687:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100395.804730:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100395.804761:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100395.804841:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254464
00000100:00100000:2.0:1766100395.804908:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3005
00000100:00100000:2.0:1766100395.804915:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417254464:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100395.804996:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045df1f7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417254464:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 81us (266us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100395.805005:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3005
00000100:00100000:1.0:1766100395.805512:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254400
00000100:00100000:1.0:1766100395.805547:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2032
00000100:00100000:1.0:1766100395.805552:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417254400:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100395.805594:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254528
00000100:00100000:3.0:1766100395.805627:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1885
00000100:00100000:3.0:1766100395.805632:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417254528:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100395.805741:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100395.806729:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417254528:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1093us (1964us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100395.806746:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1885
00000100:00100000:1.0:1766100395.806766:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000066c03f02 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417254400:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1211us (2084us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100395.806775:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2032
00010000:00010000:1.0:1766100396.029963:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100396.477797:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432987712:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100396.477871:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:0.0:1766100396.478736:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1766100396.478742:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:0.0:1766100396.478745:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:0.0:1766100396.478751:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4899
00000100:00080000:0.0:1766100396.478754:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:0.0:1766100396.478759:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100396.478781:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000007d494536 x1851885432987840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100396.478905:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987712
00000100:00100000:2.0:1766100396.478965:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2033
00000100:00100000:2.0:1766100396.478972:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432987712:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100396.479022:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001674aff2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432987712:12345-0@lo:400:kworker/u8:1.0 Request processed in 50us (1153us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100396.479030:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2033
00000100:00100000:2.0:1766100396.479112:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432987712:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100396.479133:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100396.479141:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086b72066 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432987968:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100396.479169:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100396.479196:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987968
00000100:00100000:2.0:1766100396.479220:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1280
00000100:00100000:2.0:1766100396.479224:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432987968:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100396.479255:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432987968:12345-0@lo:400:kworker/u8:1.0 Request processed in 31us (86us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100396.479262:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1280
00010000:00080000:2.0:1766100396.479306:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000086b72066 x1851885432987968/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100403 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:2.0:1766100396.479317:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086b72066 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432987968:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100396.479453:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432987776:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100396.479509:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100396.479540:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100396.479546:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432987904:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100396.479569:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100396.479610:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d494536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432987840:0@lo:8:kworker/u8:1.0
00000100:00100000:1.0:1766100396.479641:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100396.479677:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987776
00000100:00100000:0.0:1766100396.479683:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987840
00000100:00100000:0.0:1766100396.479709:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1886
00000100:00100000:1.0:1766100396.479712:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432987904
00000100:00100000:0.0:1766100396.479714:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fbaeeddc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432987840:12345-0@lo:8:kworker/u8:1.0
00000100:00100000:1.0:1766100396.479734:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3006
00000100:00100000:1.0:1766100396.479739:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432987776:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100396.479772:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dd0a2da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432987776:12345-0@lo:400:kworker/u8:1.0 Request processed in 32us (264us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100396.479778:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3006
00000100:00100000:1.0:1766100396.479791:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3007
00000100:00100000:1.0:1766100396.479795:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071bbd8cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432987904:12345-0@lo:400:kworker/u8:1.0
00010000:02020000:0.0:1766100396.479820:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100396.479820:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071bbd8cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432987904:12345-0@lo:400:kworker/u8:1.0 Request processed in 23us (249us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100396.479829:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3007
00000100:00100000:1.0:1766100396.479862:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2dfefc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432987904:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100396.479867:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fbaeeddc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432987840:12345-0@lo:8:kworker/u8:1.0 Request processed in 151us (226us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100396.479874:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1886
00000100:00100000:1.0:1766100396.479877:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432987776:0@lo:400:kworker/u8:1.0
00000100:00080000:1.0:1766100396.479908:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100396.479911:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100396.479927:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d494536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432987840:0@lo:8:kworker/u8:1.0
00010000:00010000:2.0:1766100397.053948:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100398.078942:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100399.102149:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100400.127224:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100400.925721:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100400.927781:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254720
00000100:00100000:2.0:1766100400.927841:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1887
00000100:00100000:2.0:1766100400.927848:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417254720:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100400.927959:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100400.928891:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417254720:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 1031us (3166us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100400.928970:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1887
00000100:00100000:0.0:1766100400.935318:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100400.935370:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100400.936608:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254592
00000100:00100000:2.0:1766100400.936682:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2034
00000100:00100000:2.0:1766100400.936690:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417254592:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100400.936759:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f850ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417254592:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 68us (1447us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100400.936773:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2034
00000100:00100000:1.0:1766100400.938121:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254656
00000100:00100000:3.0:1766100400.939208:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3008
00000100:00100000:3.0:1766100400.939217:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b47533c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417254656:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100400.939297:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b47533c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417254656:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 80us (3927us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100400.939304:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3008
00010000:00010000:2.0:1766100401.150630:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100401.599239:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432988032:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100401.599310:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100401.599668:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988032
00000100:00100000:0.0:1766100401.599713:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2035
00000100:00100000:0.0:1766100401.599718:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432988032:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100401.599938:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100401.599943:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100401.599947:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100401.599951:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4904
00000100:00080000:1.0:1766100401.599954:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100401.599957:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100401.599977:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000074406767 x1851885432988160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100401.600835:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e174d09e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432988032:12345-0@lo:400:kworker/u8:3.0 Request processed in 1112us (1522us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100401.600847:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2035
00000100:00100000:0.0:1766100401.600927:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432988096:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100401.600967:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100401.601007:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047cad513 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432988224:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100401.601042:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000074406767 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432988160:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100401.601070:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100401.601078:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432988288:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100401.601108:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100401.601108:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988096
00000100:00100000:1.0:1766100401.601146:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3009
00000100:00100000:1.0:1766100401.601152:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432988096:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100401.601156:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988160
00000100:00100000:0.0:1766100401.601178:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988288
00000100:00100000:1.0:1766100401.601198:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ac1f2ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432988096:12345-0@lo:400:kworker/u8:3.0 Request processed in 46us (231us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100401.601204:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1281
00000100:00100000:1.0:1766100401.601207:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3009
00000100:00100000:0.0:1766100401.601210:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432988288:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100401.601230:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1888
00000100:00100000:1.0:1766100401.601235:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7b9f638 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432988160:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100401.601250:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432988288:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (142us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100401.601257:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1281
00000100:00100000:0.0:1766100401.601277:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00010000:00080000:0.0:1766100401.601314:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008d577608 x1851885432988288/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100408 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00010000:02020000:1.0:1766100401.601328:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100401.601333:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432988288:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100401.601369:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7b9f638 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3007:x1851885432988160:12345-0@lo:8:kworker/u8:3.0 Request processed in 132us (298us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100401.601376:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000addcb14d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432988096:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100401.601377:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1888
00000100:00100000:0.0:1766100401.601403:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988224
00000100:00080000:1.0:1766100401.601417:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100401.601422:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100401.601426:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000074406767 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432988160:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100401.601432:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3010
00000100:00100000:0.0:1766100401.601436:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432988224:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100401.601472:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2dfefc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432988224:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (195us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100401.601500:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3010
00000100:00100000:0.0:1766100401.601527:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047cad513 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432988224:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100401.601553:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9408511 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432988032:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100402.173680:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100403.203166:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100404.222824:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100405.245650:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100406.044593:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100406.044649:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100406.044733:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100406.044807:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254784
00000100:00100000:0.0:1766100406.044830:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254848
00000100:00100000:2.0:1766100406.044869:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2036
00000100:00100000:2.0:1766100406.044876:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417254784:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100406.044884:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3011
00000100:00100000:0.0:1766100406.044890:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000449bd1d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417254848:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100406.044947:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006078f90f pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417254784:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 71us (214us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100406.044955:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2036
00000100:00100000:0.0:1766100406.045131:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000449bd1d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417254848:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 239us (480us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100406.045139:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3011
00000100:00100000:1.0:1766100406.046059:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254912
00000100:00100000:1.0:1766100406.046104:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1889
00000100:00100000:1.0:1766100406.046109:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081748fb8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417254912:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100406.046203:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100406.046297:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081748fb8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417254912:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 186us (1711us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100406.046305:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1889
00010000:00010000:2.0:1766100406.269713:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100406.718132:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100406.718139:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100406.718143:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100406.718150:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4910
00000100:00080000:1.0:1766100406.718153:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100406.718159:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100406.718177:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000047cad513 x1851885432988480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100406.718229:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432988608:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718295:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100406.718341:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:1.0:1766100406.718346:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432988352:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718377:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100406.718403:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432988416:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718429:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100406.718446:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100406.718450:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432988544:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718470:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100406.718517:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047cad513 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432988480:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718548:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100406.718582:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988608
00000100:00100000:1.0:1766100406.718618:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988480
00000100:00100000:1.0:1766100406.718635:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1282
00000100:00100000:1.0:1766100406.718640:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432988608:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718684:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432988608:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (391us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100406.718691:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1282
00000100:00100000:1.0:1766100406.718707:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1890
00000100:00100000:1.0:1766100406.718710:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432988480:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100406.718806:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100406.718851:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8b37342 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432988480:12345-0@lo:8:kworker/u8:3.0 Request processed in 139us (303us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100406.718857:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1890
00000100:00100000:1.0:1766100406.718876:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988416
00000100:00100000:1.0:1766100406.718897:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988544
00000100:00100000:1.0:1766100406.718912:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3012
00000100:00100000:1.0:1766100406.718915:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3e125c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432988416:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718947:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3e125c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432988416:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (517us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100406.718956:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3012
00000100:00100000:1.0:1766100406.718966:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3013
00000100:00100000:1.0:1766100406.718969:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000053dd58eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432988544:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.718991:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000053dd58eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432988544:12345-0@lo:400:kworker/u8:3.0 Request processed in 21us (522us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100406.718997:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3013
00000100:00100000:1.0:1766100406.719035:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988352
00000100:00100000:1.0:1766100406.719054:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037
00000100:00100000:1.0:1766100406.719057:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432988352:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.719080:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075eebe6e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432988352:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (703us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100406.719085:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037
00000100:00080000:1.0:1766100406.719117:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100406.719121:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100406.719124:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047cad513 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432988480:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100406.719146:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432988544:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.719154:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432988416:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100406.719171:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432988352:0@lo:400:kworker/u8:3.0
00010000:00080000:1.0:1766100406.719180:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000094636470 x1851885432988608/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100413 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100406.719189:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432988608:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100407.293830:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100408.317739:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100409.342089:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100410.367277:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100411.170454:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100411.170771:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100411.170834:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100411.174241:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255104
00000100:00100000:0.0:1766100411.174337:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1891
00000100:00100000:0.0:1766100411.174346:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417255104:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:0.0:1766100411.174454:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100411.174593:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417255104:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 247us (3758us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100411.174602:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1891
00000100:00100000:2.0:1766100411.175228:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417254976
00000100:00100000:2.0:1766100411.175268:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2038
00000100:00100000:2.0:1766100411.175273:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417254976:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100411.175317:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620a2e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417254976:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 44us (4548us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100411.175323:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2038
00000100:00100000:1.0:1766100411.177934:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255040
00000100:00100000:1.0:1766100411.178532:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3014
00000100:00100000:1.0:1766100411.178540:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073f4fbbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417255040:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100411.178711:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073f4fbbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417255040:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 170us (8271us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100411.178720:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3014
00010000:00010000:1.0:1766100411.389844:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100411.840155:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100411.840162:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100411.840164:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100411.840170:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4915
00000100:00100000:0.0:1766100411.840171:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432988672:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100411.840173:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100411.840178:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100411.840197:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000369e20f7 x1851885432988800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100411.840235:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100411.840235:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432988864:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100411.840265:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100411.840266:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100411.840271:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432988928:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100411.840298:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100411.840301:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100411.840303:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432988736:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100411.840327:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100411.840336:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988672
00000100:00100000:1.0:1766100411.840349:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988864
00000100:00100000:1.0:1766100411.840371:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988736
00000100:00100000:0.0:1766100411.840373:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2039
00000100:00100000:0.0:1766100411.840377:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432988672:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100411.840385:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3015
00000100:00100000:1.0:1766100411.840387:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047cad513 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432988864:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100411.840418:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34326f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432988672:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (184us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100411.840418:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047cad513 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432988864:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (152us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100411.840425:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2039
00000100:00100000:1.0:1766100411.840426:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3015
00000100:00100000:1.0:1766100411.840439:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3016
00000100:00100000:1.0:1766100411.840441:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432988736:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100411.840454:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988928
00000100:00100000:1.0:1766100411.840462:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c63c9d57 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432988736:12345-0@lo:400:kworker/u8:3.0 Request processed in 20us (136us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100411.840467:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3016
00000100:00100000:1.0:1766100411.840504:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432988800:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100411.840528:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100411.840554:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988800
00000100:00100000:0.0:1766100411.841199:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1283
00000100:00100000:0.0:1766100411.841208:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432988928:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100411.841223:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1892
00000100:00100000:1.0:1766100411.841226:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432988800:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100411.841246:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432988928:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (945us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100411.841252:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1283
00010000:00080000:0.0:1766100411.841291:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000008d577608 x1851885432988928/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100418 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100411.841304:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d577608 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432988928:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100411.841314:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632c907e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432988672:0@lo:400:kworker/u8:3.0
00010000:02020000:1.0:1766100411.841368:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100411.841372:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094636470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432988736:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100411.841381:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000319ec734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432988864:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100411.841401:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cb5390f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432988800:12345-0@lo:8:kworker/u8:3.0 Request processed in 173us (873us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100411.841407:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1892
00000100:00080000:0.0:1766100411.841416:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100411.841420:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100411.841423:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000369e20f7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432988800:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766100412.413707:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100413.441233:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100414.461714:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100415.485820:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100416.287789:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100416.287850:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100416.287955:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100416.288283:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255232
00000100:00100000:0.0:1766100416.288360:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3017
00000100:00100000:0.0:1766100416.288369:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417255232:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100416.288473:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a3c97aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417255232:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 104us (703us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100416.288522:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3017
00000100:00100000:3.0:1766100416.288528:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255296
00000100:00100000:3.0:1766100416.288563:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1893
00000100:00100000:3.0:1766100416.288568:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003204340c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417255296:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:3.0:1766100416.288670:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100416.288702:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003204340c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417255296:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 134us (749us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100416.288709:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1893
00000100:00100000:2.0:1766100416.288885:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255168
00000100:00100000:2.0:1766100416.288921:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2040
00000100:00100000:2.0:1766100416.288925:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417255168:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100416.288952:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba3292ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417255168:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 27us (1104us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100416.288958:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2040
00010000:00010000:2.0:1766100416.512572:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100416.957751:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00080000:1.0:1766100416.957762:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100416.957779:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100416.957785:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100416.957796:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4920
00000100:00080000:1.0:1766100416.957800:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100416.957808:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100416.957813:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014b7c3cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432988992:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.957840:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000084112edb x1851885432989120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100416.957885:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c7015c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432989184:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100416.957911:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100416.957920:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100416.957944:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100416.957949:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432989056:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100416.957951:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432989248:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.957970:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100416.957975:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100416.957997:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084112edb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432989120:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100416.958017:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432988992
00000100:00100000:1.0:1766100416.958021:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100416.958050:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989184
00000100:00100000:2.0:1766100416.958071:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2041
00000100:00100000:1.0:1766100416.958076:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989056
00000100:00100000:2.0:1766100416.958078:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432988992:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.958096:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3018
00000100:00100000:1.0:1766100416.958100:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432989184:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.958131:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008da07da1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432989184:12345-0@lo:400:kworker/u8:3.0 Request processed in 29us (209us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100416.958131:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b530ee97 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432988992:12345-0@lo:400:kworker/u8:3.0 Request processed in 52us (221us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100416.958139:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3018
00000100:00100000:2.0:1766100416.958139:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2041
00000100:00100000:1.0:1766100416.958157:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3019
00000100:00100000:1.0:1766100416.958160:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432989056:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.958186:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b5b6926 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432989056:12345-0@lo:400:kworker/u8:3.0 Request processed in 25us (215us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100416.958191:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3019
00000100:00100000:1.0:1766100416.958246:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069d150bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432989056:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.958256:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c7015c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432989184:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100416.958285:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989248
00000100:00100000:1.0:1766100416.958305:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989120
00000100:00100000:1.0:1766100416.958319:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1284
00000100:00100000:1.0:1766100416.958330:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432989248:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100416.958330:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1894
00000100:00100000:2.0:1766100416.958334:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432989120:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100416.958358:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3eafafb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432989248:12345-0@lo:400:kworker/u8:3.0 Request processed in 28us (382us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100416.958364:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1284
00010000:02020000:2.0:1766100416.958432:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00080000:1.0:1766100416.960271:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100416.960286:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100416.960294:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084112edb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432989120:0@lo:8:kworker/u8:3.0
00010000:00080000:3.0:1766100416.961085:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000099002e71 x1851885432989248/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100423 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100416.961105:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432989120:12345-0@lo:8:kworker/u8:3.0 Request processed in 2764us (3077us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100416.961107:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432989248:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100416.961126:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1894
00000100:00100000:3.0:1766100416.961130:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014b7c3cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432988992:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100417.533993:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100418.560740:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100419.581764:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100420.605703:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100421.406007:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100421.406232:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100421.406624:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100421.406676:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255424
00000100:00100000:1.0:1766100421.406766:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3020
00000100:00100000:1.0:1766100421.406774:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417255424:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100421.406780:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255360
00000100:00100000:2.0:1766100421.406837:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2042
00000100:00100000:2.0:1766100421.406845:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417255360:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100421.406866:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094636470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417255424:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 90us (633us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100421.406876:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3020
00000100:00100000:2.0:1766100421.406910:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f115299c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417255360:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 63us (916us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100421.406920:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2042
00000100:00100000:3.0:1766100421.407107:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255488
00000100:00100000:3.0:1766100421.407148:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1895
00000100:00100000:3.0:1766100421.407153:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000632c907e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417255488:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100421.407261:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100421.407425:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000632c907e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417255488:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 270us (802us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100421.407433:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1895
00010000:00010000:2.0:1766100421.632579:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100422.077790:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100422.077800:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100422.077805:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100422.077818:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4925
00000100:00080000:2.0:1766100422.077823:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100422.077831:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100422.077861:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432989376:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100422.077864:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000706cdd21 x1851885432989440/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100422.077967:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100422.077987:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432989312:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100422.078014:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:0.0:1766100422.078025:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100422.078028:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432989504:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100422.078045:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100422.078052:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432989568:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100422.078060:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100422.078085:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100422.078110:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989568
00000100:00100000:3.0:1766100422.078160:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1285
00000100:00100000:3.0:1766100422.078165:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432989568:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100422.078248:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432989440:0@lo:8:kworker/u8:1.0
00000100:00100000:1.0:1766100422.078276:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100422.078303:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989376
00000100:00100000:1.0:1766100422.078323:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989504
00000100:00100000:1.0:1766100422.078337:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3021
00000100:00100000:1.0:1766100422.078341:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432989376:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100422.078378:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc97a667 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432989376:12345-0@lo:400:kworker/u8:1.0 Request processed in 35us (414us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100422.078385:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3021
00000100:00100000:1.0:1766100422.078400:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3022
00000100:00100000:1.0:1766100422.078404:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000449764d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432989504:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100422.078434:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000449764d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432989504:12345-0@lo:400:kworker/u8:1.0 Request processed in 30us (376us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100422.078439:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3022
00000100:00100000:1.0:1766100422.078455:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989440
00000100:00100000:1.0:1766100422.078471:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896
00000100:00100000:1.0:1766100422.078497:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432989440:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:1.0:1766100422.078590:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100422.078616:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432989440:12345-0@lo:8:kworker/u8:1.0 Request processed in 141us (340us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100422.078622:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896
00000100:00080000:1.0:1766100422.078866:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1766100422.078870:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:1.0:1766100422.078874:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432989440:0@lo:8:kworker/u8:1.0
00000100:00100000:1.0:1766100422.078899:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432989504:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100422.078907:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432989376:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100422.078997:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000369e20f7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432989568:12345-0@lo:400:kworker/u8:1.0 Request processed in 830us (913us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100422.079005:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1285
00000100:00100000:2.0:1766100422.079069:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989312
00000100:00100000:2.0:1766100422.079133:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2043
00000100:00100000:2.0:1766100422.079140:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432989312:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100422.079217:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0a34911 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432989312:12345-0@lo:400:kworker/u8:1.0 Request processed in 77us (1192us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100422.079225:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2043
00010000:00080000:1.0:1766100422.079311:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b6bb6a94 x1851885432989568/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100429 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:1.0:1766100422.079322:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432989568:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100422.079332:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432989312:0@lo:400:kworker/u8:1.0
00010000:00010000:2.0:1766100422.654255:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100423.679846:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100424.701763:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100425.726515:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100426.526288:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100426.526338:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100426.526398:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255552
00000100:00100000:3.0:1766100426.526506:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2044
00000100:00100000:3.0:1766100426.526513:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417255552:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100426.526597:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f645cd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417255552:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 83us (321us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100426.526606:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2044
00000100:00100000:2.0:1766100426.527461:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255616
00000100:00100000:2.0:1766100426.527550:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3023
00000100:00100000:2.0:1766100426.527558:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417255616:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100426.527661:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078059cfe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417255616:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 103us (1324us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100426.527671:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3023
00000100:00100000:1.0:1766100426.528118:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100426.529430:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255680
00000100:00100000:2.0:1766100426.529525:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1897
00000100:00100000:2.0:1766100426.529533:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417255680:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100426.529725:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100426.529820:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417255680:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 287us (1704us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100426.529828:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1897
00010000:00010000:1.0:1766100426.750167:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1766100427.197770:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100427.197781:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100427.197786:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100427.197797:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4930
00000100:00080000:2.0:1766100427.197802:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100427.197809:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100427.197837:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000b6bb6a94 x1851885432989760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100427.197857:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432989632:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100427.197947:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100427.197956:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432989696:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100427.197999:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100427.198031:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100427.198038:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432989824:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100427.198059:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100427.198106:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989696
00000100:00100000:3.0:1766100427.198142:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989824
00000100:00100000:3.0:1766100427.198160:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3024
00000100:00100000:3.0:1766100427.198164:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6d2bcd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432989696:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100427.198213:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6d2bcd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432989696:12345-0@lo:400:kworker/u8:1.0 Request processed in 47us (213us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100427.198222:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3024
00000100:00100000:3.0:1766100427.198237:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3025
00000100:00100000:3.0:1766100427.198241:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000965f5dd0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432989824:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100427.198264:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000965f5dd0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432989824:12345-0@lo:400:kworker/u8:1.0 Request processed in 22us (204us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100427.198269:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3025
00000100:00100000:3.0:1766100427.198299:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048b52776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432989824:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100427.198309:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bdf5e63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432989696:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100427.199961:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100427.199975:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f7f92201 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432989888:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100427.200033:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100427.200368:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989632
00000100:00100000:2.0:1766100427.200407:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2045
00000100:00100000:2.0:1766100427.200412:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432989632:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100427.200450:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bce084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432989632:12345-0@lo:400:kworker/u8:1.0 Request processed in 38us (2506us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100427.200456:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2045
00000100:00100000:0.0:1766100427.202091:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432989632:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100427.202156:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432989760:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100427.202197:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100427.203625:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989888
00000100:00100000:3.0:1766100427.203679:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989760
00000100:00100000:3.0:1766100427.203710:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1286
00000100:00100000:3.0:1766100427.203715:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000319ec734 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432989888:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100427.203789:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000319ec734 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432989888:12345-0@lo:400:kworker/u8:1.0 Request processed in 73us (3756us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100427.203796:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1286
00000100:00100000:3.0:1766100427.203813:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1898
00000100:00100000:3.0:1766100427.203816:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432989760:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:3.0:1766100427.203904:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100427.203938:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d577608 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432989760:12345-0@lo:8:kworker/u8:1.0 Request processed in 120us (1740us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100427.203944:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1898
00000100:00080000:3.0:1766100427.203979:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100427.203983:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100427.203988:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432989760:0@lo:8:kworker/u8:1.0
00010000:00080000:3.0:1766100427.204020:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000f7f92201 x1851885432989888/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100434 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:3.0:1766100427.204030:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f7f92201 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432989888:0@lo:400:kworker/u8:1.0
00010000:00010000:2.0:1766100427.774305:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100428.798647:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100429.821702:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100430.846722:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100431.645052:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100431.645110:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100431.645167:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255808
00000100:00100000:2.0:1766100431.645197:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255744
00000100:00100000:1.0:1766100431.645228:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3026
00000100:00100000:1.0:1766100431.645235:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417255808:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:2.0:1766100431.645251:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2046
00000100:00100000:2.0:1766100431.645257:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417255744:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:1.0:1766100431.645314:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9408511 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417255808:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 78us (203us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100431.645323:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3026
00000100:00100000:2.0:1766100431.645514:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf74a08d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417255744:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 255us (467us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100431.645522:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2046
00000100:00100000:0.0:1766100431.652673:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100431.652753:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255872
00000100:00100000:1.0:1766100431.652793:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1899
00000100:00100000:1.0:1766100431.652799:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417255872:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100431.652888:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100431.652953:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000074406767 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417255872:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 153us (296us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100431.652960:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1899
00010000:00010000:2.0:1766100431.869787:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100432.317764:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100432.318571:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c41f2c96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432989952:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100432.318645:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100432.318813:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432989952
00000100:00100000:0.0:1766100432.319350:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2047
00000100:00100000:0.0:1766100432.319357:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432989952:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100432.319414:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000625a6369 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432989952:12345-0@lo:400:kworker/u8:1.0 Request processed in 57us (770us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100432.319421:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2047
00000100:00100000:3.0:1766100432.319501:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c41f2c96 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432989952:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100432.320697:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad90c4a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432990016:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100432.320746:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766100432.323270:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100432.323280:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100432.323283:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100432.323291:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4935
00000100:00080000:2.0:1766100432.323294:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100432.323300:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100432.323336:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002c6fd5dd x1851885432990080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100432.323389:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c6fd5dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432990080:0@lo:8:kworker/u8:1.0
00000100:00100000:3.0:1766100432.323449:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100432.323532:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990080
00000100:00100000:0.0:1766100432.323568:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900
00000100:00100000:0.0:1766100432.323574:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003fede54a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432990080:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:0.0:1766100432.323677:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100432.324512:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990016
00000100:00100000:1.0:1766100432.324555:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3027
00000100:00100000:1.0:1766100432.324560:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004833a3bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432990016:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100432.324624:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004833a3bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432990016:12345-0@lo:400:kworker/u8:1.0 Request processed in 62us (3876us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100432.324630:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3027
00000100:00100000:1.0:1766100432.324836:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad90c4a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432990016:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100432.325065:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432990208:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100432.325101:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100432.325136:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990208
00000100:00100000:2.0:1766100432.325163:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1287
00000100:00100000:2.0:1766100432.325166:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef33bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432990208:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100432.325198:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef33bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432990208:12345-0@lo:400:kworker/u8:1.0 Request processed in 31us (97us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100432.325203:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1287
00010000:00080000:2.0:1766100432.325231:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000e96bbcf1 x1851885432990208/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100439 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:2.0:1766100432.325242:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e96bbcf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432990208:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100432.325254:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:2.0:1766100432.325259:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432990144:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100432.325281:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100432.325300:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990144
00000100:00100000:2.0:1766100432.325318:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3028
00000100:00100000:2.0:1766100432.325321:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e96bbcf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432990144:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100432.325347:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e96bbcf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432990144:12345-0@lo:400:kworker/u8:1.0 Request processed in 25us (67us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100432.325352:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3028
00000100:00100000:2.0:1766100432.325370:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432990144:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100432.326173:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003fede54a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3007:x1851885432990080:12345-0@lo:8:kworker/u8:1.0 Request processed in 2596us (2722us total) trans 0 rc -19/-19
00000100:00080000:3.0:1766100432.326608:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100432.326616:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100432.326620:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c6fd5dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432990080:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100432.326699:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900
00010000:00010000:1.0:1766100432.894321:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100433.917789:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100434.941650:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100435.965735:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100436.768383:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100436.769718:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100436.769775:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256000
00000100:00100000:2.0:1766100436.769807:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417255936
00000100:00100000:1.0:1766100436.769841:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3029
00000100:00100000:2.0:1766100436.769845:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2048
00000100:00100000:1.0:1766100436.769852:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417256000:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100436.769854:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417255936:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100436.769935:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000707ba173 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417255936:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 82us (1559us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100436.769942:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2048
00000100:00100000:1.0:1766100436.770639:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000addcb14d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417256000:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 788us (923us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100436.770648:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3029
00000100:00100000:0.0:1766100436.770676:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100436.770707:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256064
00000100:00100000:1.0:1766100436.770741:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1901
00000100:00100000:1.0:1766100436.770746:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417256064:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:1.0:1766100436.770839:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100436.770870:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af55eb76 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417256064:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 124us (197us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100436.770877:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1901
00010000:00010000:2.0:1766100436.990958:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100437.437944:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432990272:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100437.438043:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00080000:2.0:1766100437.438660:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1766100437.438667:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:2.0:1766100437.438671:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:2.0:1766100437.438677:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4940
00000100:00080000:2.0:1766100437.438681:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:2.0:1766100437.438687:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:2.0:1766100437.438713:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000094cabff9 x1851885432990400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:1.0:1766100437.438746:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:1.0:1766100437.438764:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432990464:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.438813:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100437.438836:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eb9fd8f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432990336:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.438858:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100437.438905:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990464
00000100:00100000:1.0:1766100437.438960:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990336
00000100:00100000:1.0:1766100437.438984:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3030
00000100:00100000:1.0:1766100437.438990:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432990464:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439048:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e485a7e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432990464:12345-0@lo:400:kworker/u8:3.0 Request processed in 57us (235us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100437.439059:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3030
00000100:00100000:1.0:1766100437.439079:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3031
00000100:00100000:1.0:1766100437.439082:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432990336:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439109:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000150fb02 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432990336:12345-0@lo:400:kworker/u8:3.0 Request processed in 26us (251us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100437.439115:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3031
00000100:00100000:1.0:1766100437.439161:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d8990d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432990464:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439171:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eb9fd8f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432990336:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439217:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432990528:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439245:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100437.439257:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990272
00000100:00100000:1.0:1766100437.439279:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990528
00000100:00100000:3.0:1766100437.439293:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2049
00000100:00100000:3.0:1766100437.439299:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432990272:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439310:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1288
00000100:00100000:1.0:1766100437.439315:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432990528:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100437.439355:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cfcb09a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432990272:12345-0@lo:400:kworker/u8:3.0 Request processed in 55us (1312us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100437.439361:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432990528:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (114us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100437.439368:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2049
00000100:00100000:1.0:1766100437.439372:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1288
00010000:00080000:1.0:1766100437.439409:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000871971ee x1851885432990528/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100444 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:3.0:1766100437.439415:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094cabff9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432990400:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439426:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000871971ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432990528:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100437.439448:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100437.439504:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f28b7aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432990272:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100437.439505:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990400
00000100:00100000:1.0:1766100437.439527:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1902
00000100:00100000:1.0:1766100437.439532:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432990400:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100437.439638:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100437.439683:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6bb6a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432990400:12345-0@lo:8:kworker/u8:3.0 Request processed in 150us (235us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100437.439689:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1902
00000100:00080000:3.0:1766100437.440083:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100437.440089:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100437.440093:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094cabff9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432990400:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766100438.014264:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100439.049897:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100440.061828:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100441.085762:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100441.887914:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100441.888062:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256128
00000100:00100000:0.0:1766100441.888148:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2050
00000100:00100000:0.0:1766100441.888157:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417256128:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100441.888257:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec0f8859 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417256128:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 101us (357us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100441.888266:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2050
00000100:00100000:0.0:1766100441.898522:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100441.898653:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100441.898885:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256192
00000100:00100000:3.0:1766100441.898953:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3032
00000100:00100000:3.0:1766100441.898961:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417256192:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100441.899064:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea384f3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417256192:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 102us (410us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100441.899083:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3032
00000100:00100000:2.0:1766100441.899519:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256256
00000100:00100000:2.0:1766100441.899564:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1903
00000100:00100000:2.0:1766100441.899570:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417256256:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:2.0:1766100441.899675:0:6303:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100441.899736:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0edbad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2227:x1851885417256256:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 165us (1264us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100441.899743:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1903
00010000:00010000:1.0:1766100442.111647:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100442.560161:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100442.560169:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100442.560173:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1766100442.560184:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432990592:0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100442.560191:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4945
00000100:00080000:1.0:1766100442.560194:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100442.560200:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100442.560225:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000706cdd21 x1851885432990720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100442.560277:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100442.560317:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:3.0:1766100442.560322:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084112edb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432990848:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100442.560350:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100442.560385:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990592
00000100:00100000:1.0:1766100442.560429:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432990656:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100442.560436:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2051
00000100:00100000:3.0:1766100442.560443:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432990592:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100442.560461:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100442.560502:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100442.560507:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432990784:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100442.560516:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000609c28ad pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432990592:12345-0@lo:400:kworker/u8:3.0 Request processed in 73us (240us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100442.560524:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2051
00000100:00100000:1.0:1766100442.560539:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100442.560565:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432990720:0@lo:8:kworker/u8:3.0
00000100:00100000:3.0:1766100442.560573:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432990592:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100442.560599:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100442.560620:0:8561:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990656
00000100:00100000:1.0:1766100442.560626:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990784
00000100:00100000:3.0:1766100442.560769:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3033
00000100:00100000:3.0:1766100442.560774:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432990784:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100442.560798:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990848
00000100:00100000:3.0:1766100442.560811:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014b7c3cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432990784:12345-0@lo:400:kworker/u8:3.0 Request processed in 37us (271us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100442.560817:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3033
00000100:00100000:2.0:1766100442.560825:0:6303:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990720
00000100:00100000:3.0:1766100442.560827:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3034
00000100:00100000:3.0:1766100442.560830:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432990656:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100442.560842:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1289
00000100:00100000:2.0:1766100442.560845:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039f6e93a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432990848:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100442.560854:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c7015c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432990656:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (392us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100442.560859:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3034
00000100:00100000:3.0:1766100442.560878:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1904
00000100:00100000:2.0:1766100442.560880:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039f6e93a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432990848:12345-0@lo:400:kworker/u8:3.0 Request processed in 34us (530us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100442.560881:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432990720:12345-0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100442.560886:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1289
00010000:00080000:2.0:1766100442.560918:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000084112edb x1851885432990848/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100449 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:2.0:1766100442.560929:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084112edb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432990848:0@lo:400:kworker/u8:3.0
00010000:02020000:3.0:1766100442.560982:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100442.561031:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432990720:12345-0@lo:8:kworker/u8:3.0 Request processed in 148us (431us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100442.561037:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1904
00000100:00080000:2.0:1766100442.561105:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100442.561109:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100442.561113:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432990720:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100442.561576:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099002e71 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432990784:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100442.561600:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432990656:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100443.133792:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100444.157880:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100445.181781:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100446.206591:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:0.0:1766100447.004711:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100447.007568:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100447.007648:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100447.007793:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256384
00000100:00100000:3.0:1766100447.007866:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3035
00000100:00100000:3.0:1766100447.007874:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417256384:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100447.007967:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cd88d9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:56772462-1282-4783-8e1a-69be7ca0dd12+8:2231:x1851885417256384:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 93us (320us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100447.007975:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3035
00000100:00100000:1.0:1766100447.008790:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256320
00000100:00100000:1.0:1766100447.008846:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2052
00000100:00100000:1.0:1766100447.008851:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417256320:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100447.008895:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e05b57d1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2231:x1851885417256320:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 44us (1351us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100447.008902:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2052
00000100:00100000:2.0:1766100447.009052:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256448
00000100:00100000:2.0:1766100447.009103:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1905
00000100:00100000:2.0:1766100447.009108:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417256448:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100447.009208:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100447.009385:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a733f90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417256448:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 276us (4686us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100447.009394:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1905
00010000:00010000:2.0:1766100447.232173:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100447.678771:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432990912:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100447.678894:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100447.678982:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990912
00000100:00100000:2.0:1766100447.679045:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2053
00000100:00100000:2.0:1766100447.679052:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432990912:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100447.679114:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000664994a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432990912:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (225us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100447.679123:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2053
00000100:00100000:2.0:1766100447.679177:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432990912:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100447.679554:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00080000:1.0:1766100447.680915:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100447.680926:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100447.680930:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100447.680940:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4951
00000100:00080000:1.0:1766100447.680944:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:1.0:1766100447.680951:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100447.680993:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000678c16b3 x1851885432991040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:2.0:1766100447.681086:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432991040:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100447.681165:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100447.682799:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000914f119b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432990976:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100447.682880:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100447.682960:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432990976
00000100:00100000:3.0:1766100447.683016:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3036
00000100:00100000:3.0:1766100447.683022:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432990976:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100447.683083:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bdf5e63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432990976:12345-0@lo:400:kworker/u8:3.0 Request processed in 60us (205us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100447.683092:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3036
00000100:00100000:3.0:1766100447.683144:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000914f119b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432990976:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100447.683192:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991040
00000100:00100000:3.0:1766100447.683217:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906
00000100:00100000:3.0:1766100447.683221:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000585c6915 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432991040:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:3.0:1766100447.683345:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100447.684309:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000914f119b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432991168:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100447.684379:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100447.684426:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991168
00000100:00100000:1.0:1766100447.684468:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1290
00000100:00100000:1.0:1766100447.684474:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432991168:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100447.684536:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000d48768b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432991168:12345-0@lo:400:kworker/u8:3.0 Request processed in 61us (157us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100447.684583:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1290
00010000:00080000:1.0:1766100447.684628:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000914f119b x1851885432991168/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100454 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100447.684641:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000914f119b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432991168:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100447.684675:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100447.684683:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432991104:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100447.684713:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100447.684740:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991104
00000100:00100000:1.0:1766100447.684768:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3037
00000100:00100000:1.0:1766100447.684771:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000914f119b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432991104:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100447.684803:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000914f119b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432991104:12345-0@lo:400:kworker/u8:3.0 Request processed in 31us (91us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100447.684809:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3037
00000100:00100000:1.0:1766100447.684832:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432991104:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100447.686389:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000585c6915 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432991040:12345-0@lo:8:kworker/u8:3.0 Request processed in 3158us (5217us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100447.686404:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906
00000100:00080000:2.0:1766100447.686533:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100447.686538:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100447.686553:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432991040:0@lo:8:kworker/u8:3.0
00010000:00010000:1.0:1766100448.255502:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100449.277849:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100450.301699:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100451.325898:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:3.0:1766100452.124660:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100452.124738:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100452.124855:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256512
00000100:00100000:2.0:1766100452.124930:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2054
00000100:00100000:2.0:1766100452.124938:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417256512:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100452.125029:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000264427b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417256512:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 91us (291us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100452.125038:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2054
00000100:00100000:1.0:1766100452.126901:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256576
00000100:00100000:0.0:1766100452.127084:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100452.127257:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3038
00000100:00100000:1.0:1766100452.127265:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417256576:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:1.0:1766100452.127351:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048b52776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417256576:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 86us (2706us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100452.127359:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3038
00000100:00100000:3.0:1766100452.128240:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256640
00000100:00100000:3.0:1766100452.128276:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1907
00000100:00100000:3.0:1766100452.128282:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417256640:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100452.128380:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100452.129712:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417256640:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 1422us (2625us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100452.129733:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1907
00010000:00010000:2.0:1766100452.351028:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100452.801227:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432991232:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100452.801345:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100452.801457:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991232
00000100:00100000:2.0:1766100452.801574:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2055
00000100:00100000:2.0:1766100452.801581:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432991232:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:3.0:1766100452.803310:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100452.803319:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100452.803322:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100452.803331:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4956
00000100:00080000:3.0:1766100452.803336:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s
00000100:00080000:3.0:1766100452.803343:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100452.803376:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000076bb4d8b x1851885432991360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100452.804232:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432991488:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804333:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100452.804417:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a421d01c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432991424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804451:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100452.804473:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100452.804502:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432991296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804547:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100452.804591:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991424
00000100:00100000:0.0:1766100452.804645:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991296
00000100:00100000:0.0:1766100452.804670:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3039
00000100:00100000:0.0:1766100452.804677:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c5bf9cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432991424:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804737:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c5bf9cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432991424:12345-0@lo:400:kworker/u8:3.0 Request processed in 59us (285us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100452.804746:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3039
00000100:00100000:0.0:1766100452.804771:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3040
00000100:00100000:0.0:1766100452.804775:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432991296:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804798:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d35037c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432991296:12345-0@lo:400:kworker/u8:3.0 Request processed in 22us (251us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100452.804807:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3040
00000100:00100000:0.0:1766100452.804850:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000706cdd21 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432991296:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804859:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a421d01c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432991424:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804894:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991488
00000100:00100000:0.0:1766100452.804916:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1291
00000100:00100000:0.0:1766100452.804920:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000458dfff5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432991488:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.804961:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000458dfff5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432991488:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (630us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100452.804967:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1291
00010000:00080000:0.0:1766100452.804991:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b4d81768 x1851885432991488/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100459 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100452.805001:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432991488:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100452.805035:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432991360:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100452.805069:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100452.805092:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991360
00000100:00100000:0.0:1766100452.805105:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1908
00000100:00100000:0.0:1766100452.805109:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432991360:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:0.0:1766100452.805206:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:0.0:1766100452.805238:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad887d7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432991360:12345-0@lo:8:kworker/u8:3.0 Request processed in 128us (168us total) trans 0 rc -19/-19
00000100:00100000:0.0:1766100452.805245:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1908
00000100:00080000:0.0:1766100452.805268:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100452.805273:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100452.805278:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076bb4d8b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432991360:0@lo:8:kworker/u8:3.0
00000100:00100000:2.0:1766100452.809785:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c920ad5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432991232:12345-0@lo:400:kworker/u8:3.0 Request processed in 8188us (8426us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100452.809809:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2055
00000100:00100000:1.0:1766100452.813256:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ae602d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432991232:0@lo:400:kworker/u8:3.0
00010000:00010000:2.0:1766100453.373834:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100454.400049:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100455.421746:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100456.447076:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100457.244669:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100457.244722:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100457.244806:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100457.245432:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256768
00000100:00100000:0.0:1766100457.245562:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3041
00000100:00100000:0.0:1766100457.245569:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f484c134 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417256768:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:0.0:1766100457.245681:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f484c134 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2230:x1851885417256768:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 111us (1023us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100457.245690:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3041
00000100:00100000:3.0:1766100457.245877:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256704
00000100:00100000:3.0:1766100457.245919:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2056
00000100:00100000:3.0:1766100457.245924:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417256704:12345-192.168.203.52@tcp:400:kworker/u8:2.0
00000100:00100000:3.0:1766100457.245966:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000646e5e83 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2230:x1851885417256704:12345-192.168.203.52@tcp:400:kworker/u8:2.0 Request processed in 41us (1243us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100457.245972:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2056
00000100:00100000:1.0:1766100457.246808:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256832
00000100:00100000:1.0:1766100457.246859:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1909
00000100:00100000:1.0:1766100457.246864:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094cabff9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417256832:12345-192.168.203.52@tcp:8:kworker/u8:2.0
00010000:02020000:1.0:1766100457.246969:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100457.247028:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094cabff9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417256832:12345-192.168.203.52@tcp:8:kworker/u8:2.0 Request processed in 163us (2221us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100457.247195:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1909
00010000:00010000:1.0:1766100457.470138:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100457.917753:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100457.917763:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100457.917768:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100457.917779:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4961
00000100:00080000:3.0:1766100457.917784:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 6s
00000100:00080000:3.0:1766100457.917791:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100457.917820:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@0000000032a0d422 x1851885432991680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100457.917961:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432991808:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918047:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100457.918105:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:0.0:1766100457.918111:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f180bc23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432991552:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918142:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100457.918170:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432991744:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918197:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100457.918214:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:0.0:1766100457.918220:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432991616:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918241:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100457.918265:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432991680:0@lo:8:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918292:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100457.918332:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991744
00000100:00100000:0.0:1766100457.918384:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991616
00000100:00100000:0.0:1766100457.918405:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3042
00000100:00100000:0.0:1766100457.918413:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432991744:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918509:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076bb4d8b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432991744:12345-0@lo:400:kworker/u8:1.0 Request processed in 53us (269us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100457.918522:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3042
00000100:00100000:0.0:1766100457.918544:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3043
00000100:00100000:0.0:1766100457.918548:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432991616:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918573:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000706cdd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432991616:12345-0@lo:400:kworker/u8:1.0 Request processed in 24us (331us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100457.918579:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3043
00000100:00100000:0.0:1766100457.918610:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991552
00000100:00100000:0.0:1766100457.918635:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2057
00000100:00100000:0.0:1766100457.918639:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432991552:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918670:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2a3a17 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432991552:12345-0@lo:400:kworker/u8:1.0 Request processed in 30us (528us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100457.918677:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2057
00000100:00100000:0.0:1766100457.918728:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432991616:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918738:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432991744:0@lo:400:kworker/u8:1.0
00000100:00100000:0.0:1766100457.918771:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f180bc23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432991552:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100457.919179:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991808
00000100:00100000:2.0:1766100457.919218:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991680
00000100:00100000:2.0:1766100457.919238:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1292
00000100:00100000:2.0:1766100457.919243:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432991808:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100457.919296:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010ae602d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432991808:12345-0@lo:400:kworker/u8:1.0 Request processed in 53us (1251us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100457.919304:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1292
00000100:00100000:2.0:1766100457.919317:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1910
00000100:00100000:2.0:1766100457.919320:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432991680:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:2.0:1766100457.919423:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100457.919459:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432991680:12345-0@lo:8:kworker/u8:1.0 Request processed in 138us (1166us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100457.919497:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1910
00000100:00080000:2.0:1766100457.919541:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100457.919546:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100457.919550:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432991680:0@lo:8:kworker/u8:1.0
00010000:00080000:2.0:1766100457.919575:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000b4d81768 x1851885432991808/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100464 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:2.0:1766100457.919585:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b4d81768 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432991808:0@lo:400:kworker/u8:1.0
00010000:00010000:2.0:1766100458.504561:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100459.518646:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100460.541853:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100461.565778:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100462.366020:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100462.366354:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100462.366427:0:11110:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257024
00000100:00100000:3.0:1766100462.366528:0:11110:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1911
00000100:00100000:3.0:1766100462.366535:0:11110:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000730a3e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417257024:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100462.366633:0:11110:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100462.366706:0:11110:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000730a3e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2227:x1851885417257024:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 171us (694us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100462.366715:0:11110:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1911
00000100:00100000:0.0:1766100462.366762:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256896
00000100:00100000:0.0:1766100462.366793:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2058
00000100:00100000:0.0:1766100462.366797:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417256896:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100462.366828:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adca9c9e pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417256896:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 30us (475us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100462.366834:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2058
00000100:00100000:2.0:1766100462.367329:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100462.367431:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417256960
00000100:00100000:0.0:1766100462.368042:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3044
00000100:00100000:0.0:1766100462.368050:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f28b7aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417256960:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100462.368110:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f28b7aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417256960:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 59us (782us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100462.368117:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3044
00010000:00010000:2.0:1766100462.591219:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100463.038175:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100463.038183:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100463.038186:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100463.038194:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4966
00000100:00080000:1.0:1766100463.038198:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 11s
00000100:00080000:1.0:1766100463.038203:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100463.038226:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000678c16b3 x1851885432992000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100463.038361:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432991936:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100463.038420:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100463.038461:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:3.0:1766100463.038466:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081748fb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432992064:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100463.038504:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100463.038628:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992000:0@lo:8:kworker/u8:1.0
00000100:00100000:2.0:1766100463.038659:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100463.038699:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432991872:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100463.038730:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100463.038752:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:2.0:1766100463.038756:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432992128:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100463.038777:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:2.0:1766100463.038809:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991872
00000100:00100000:2.0:1766100463.038854:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2059
00000100:00100000:2.0:1766100463.038860:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432991872:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100463.038909:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff75190b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432991872:12345-0@lo:400:kworker/u8:1.0 Request processed in 48us (178us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100463.038916:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2059
00000100:00100000:2.0:1766100463.038964:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030ebde24 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432991872:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100463.039144:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432991936
00000100:00100000:1.0:1766100463.039179:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992064
00000100:00100000:1.0:1766100463.039193:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3045
00000100:00100000:1.0:1766100463.039196:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432991936:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100463.039231:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a421d01c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432991936:12345-0@lo:400:kworker/u8:1.0 Request processed in 34us (811us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100463.039237:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3045
00000100:00100000:1.0:1766100463.039246:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3046
00000100:00100000:1.0:1766100463.039248:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432992064:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100463.039265:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4d81768 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432992064:12345-0@lo:400:kworker/u8:1.0 Request processed in 16us (762us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100463.039269:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3046
00000100:00100000:1.0:1766100463.039293:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081748fb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432992064:0@lo:400:kworker/u8:1.0
00000100:00100000:1.0:1766100463.039300:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000d48768b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432991936:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100463.039473:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992000
00000100:00100000:3.0:1766100463.039518:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992128
00000100:00100000:3.0:1766100463.039536:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1293
00000100:00100000:3.0:1766100463.039540:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eb9fd8f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432992128:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100463.039572:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eb9fd8f1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432992128:12345-0@lo:400:kworker/u8:1.0 Request processed in 31us (795us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100463.039577:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1293
00000100:00100000:3.0:1766100463.039589:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912
00000100:00100000:3.0:1766100463.039592:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432992000:12345-0@lo:8:kworker/u8:1.0
00010000:02020000:3.0:1766100463.039683:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100463.039713:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000871971ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432992000:12345-0@lo:8:kworker/u8:1.0 Request processed in 120us (1053us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100463.039718:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912
00000100:00080000:2.0:1766100463.040421:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1766100463.040426:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:2.0:1766100463.040441:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000678c16b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992000:0@lo:8:kworker/u8:1.0
00010000:00080000:2.0:1766100463.040503:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002680be6b x1851885432992128/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100470 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:2.0:1766100463.040512:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002680be6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432992128:0@lo:400:kworker/u8:1.0
00010000:00010000:1.0:1766100463.614154:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100464.637705:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100465.662446:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100466.686444:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100467.484532:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100467.484720:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257088
00000100:00100000:2.0:1766100467.485391:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2060
00000100:00100000:2.0:1766100467.485529:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417257088:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100467.485622:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f6fbbb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417257088:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 96us (1097us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100467.485630:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2060
00000100:00100000:2.0:1766100467.486813:0:3002:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100467.487317:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257152
00000100:00100000:1.0:1766100467.487420:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:3.0:1766100467.487565:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257216
00000100:00100000:3.0:1766100467.487629:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1913
00000100:00100000:3.0:1766100467.487636:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081748fb8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417257216:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:3.0:1766100467.487757:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:3.0:1766100467.487830:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081748fb8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417257216:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 193us (414us total) trans 0 rc -19/-19
00000100:00100000:3.0:1766100467.487837:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1913
00000100:00100000:2.0:1766100467.488439:0:8561:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3047
00000100:00100000:2.0:1766100467.488448:0:8561:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d8990d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417257152:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:2.0:1766100467.488550:0:8561:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d8990d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417257152:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 102us (1745us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100467.488558:0:8561:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3047
00010000:00010000:2.0:1766100467.710112:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1766100468.158805:0:15616:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1766100468.158812:0:15616:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:3.0:1766100468.158815:0:15616:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:3.0:1766100468.158823:0:15616:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4971
00000100:00080000:3.0:1766100468.158826:0:15616:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 16s
00000100:00080000:3.0:1766100468.158832:0:15616:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:3.0:1766100468.158850:0:15616:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002c75ca47 x1851885432992320/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100468.158890:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432992448:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.158952:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100468.158978:0:3008:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0]
00000100:00100000:3.0:1766100468.158984:0:3008:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432992192:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159013:0:3008:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100468.159037:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432992384:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159066:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100468.159084:0:3011:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3]
00000100:00100000:3.0:1766100468.159090:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f180bc23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432992256:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159116:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100468.159153:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992192
00000100:00100000:3.0:1766100468.159193:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061
00000100:00100000:3.0:1766100468.159198:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432992192:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159241:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001285fa5b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3008:x1851885432992192:12345-0@lo:400:kworker/u8:1.0 Request processed in 42us (229us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100468.159248:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061
00000100:00100000:3.0:1766100468.159278:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992384
00000100:00100000:3.0:1766100468.159300:0:10498:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992256
00000100:00100000:3.0:1766100468.159313:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3048
00000100:00100000:3.0:1766100468.159316:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432992384:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159345:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d494536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432992384:12345-0@lo:400:kworker/u8:1.0 Request processed in 27us (279us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100468.159350:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3048
00000100:00100000:3.0:1766100468.159358:0:10498:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3049
00000100:00100000:3.0:1766100468.159361:0:10498:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432992256:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159380:0:10498:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086b72066 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3011:x1851885432992256:12345-0@lo:400:kworker/u8:1.0 Request processed in 18us (267us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100468.159386:0:10498:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3049
00000100:00100000:2.0:1766100468.159399:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992320:0@lo:8:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159431:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f180bc23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3011:1851885432992256:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100468.159437:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100468.159446:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f2f7702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432992384:0@lo:400:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159506:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032a0d422 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:4d8e661b-5d90-4003-ae19-6056efb0a422:3008:1851885432992192:0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100468.159511:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992448
00000100:00100000:2.0:1766100468.159536:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992320
00000100:00100000:3.0:1766100468.159548:0:6303:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1294
00000100:00100000:3.0:1766100468.159554:0:6303:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432992448:12345-0@lo:400:kworker/u8:1.0
00000100:00100000:2.0:1766100468.159557:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1914
00000100:00100000:2.0:1766100468.159563:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432992320:12345-0@lo:8:kworker/u8:1.0
00000100:00100000:3.0:1766100468.159586:0:6303:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0591111 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3008:x1851885432992448:12345-0@lo:400:kworker/u8:1.0 Request processed in 32us (634us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100468.159592:0:6303:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1294
00010000:00080000:3.0:1766100468.159611:0:3008:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@0000000020b6667c x1851885432992448/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100475 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:1.0'
00000100:00100000:3.0:1766100468.159619:0:3008:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000020b6667c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3008:1851885432992448:0@lo:400:kworker/u8:1.0
00010000:02020000:2.0:1766100468.159656:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100468.164208:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c6fd5dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3007:x1851885432992320:12345-0@lo:8:kworker/u8:1.0 Request processed in 4635us (4765us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100468.164234:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1914
00000100:00080000:3.0:1766100468.164364:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100468.164369:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100468.164375:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c75ca47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992320:0@lo:8:kworker/u8:1.0
00010000:00010000:2.0:1766100468.739258:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100469.759961:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100470.782412:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100471.806558:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100472.604349:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100472.604408:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100472.604589:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257280
00000100:00100000:2.0:1766100472.604643:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2062
00000100:00100000:2.0:1766100472.604649:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417257280:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100472.604754:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257344
00000100:00100000:0.0:1766100472.604789:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3050
00000100:00100000:0.0:1766100472.604794:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003fede54a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417257344:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100472.604842:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003fede54a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417257344:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 46us (433us total) trans 0 rc 0/0
00000100:00100000:2.0:1766100472.604844:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000867a5966 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:d30215d4-f453-4d61-9211-cb0b369f1629+9:2229:x1851885417257280:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 191us (500us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100472.604854:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3050
00000100:00100000:2.0:1766100472.604858:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2062
00000100:00100000:1.0:1766100472.605470:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100472.606260:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257408
00000100:00100000:2.0:1766100472.606302:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1915
00000100:00100000:2.0:1766100472.606307:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417257408:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100472.606421:0:6301:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100472.606494:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002680be6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2227:x1851885417257408:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 186us (1026us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100472.606502:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1915
00010000:00010000:1.0:1766100472.829763:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1766100473.277742:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000506576fd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432992512:0@lo:400:kworker/u8:3.0
00000100:00100000:2.0:1766100473.277818:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100473.277957:0:5506:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992512
00000100:00100000:3.0:1766100473.278004:0:5506:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2063
00000100:00100000:3.0:1766100473.278010:0:5506:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432992512:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100473.278073:0:5506:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086af8ddb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432992512:12345-0@lo:400:kworker/u8:3.0 Request processed in 62us (256us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100473.278079:0:5506:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2063
00000100:00100000:0.0:1766100473.278173:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000506576fd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432992512:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100473.278496:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432992576:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100473.278528:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100473.278580:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992576
00000100:00100000:0.0:1766100473.278618:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3051
00000100:00100000:0.0:1766100473.278622:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432992576:12345-0@lo:400:kworker/u8:3.0
00000100:00080000:1.0:1766100473.279363:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100473.279369:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100473.279372:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100473.279380:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4976
00000100:00080000:1.0:1766100473.279383:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766100473.279388:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:0.0:1766100473.279407:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000020b6667c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432992576:12345-0@lo:400:kworker/u8:3.0 Request processed in 781us (876us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100473.279410:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432992576:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100473.279412:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@000000002ec3fedd x1851885432992640/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:0.0:1766100473.279424:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3051
00000100:00100000:1.0:1766100473.279928:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100473.279929:0:3011:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acd0bfef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432992704:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100473.279934:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432992768:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100473.279984:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992640:0@lo:8:kworker/u8:3.0
00000100:00100000:0.0:1766100473.280096:0:3011:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100473.280129:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992704
00000100:00100000:0.0:1766100473.280155:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3052
00000100:00100000:0.0:1766100473.280159:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432992704:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100473.280201:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7427f30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3011:x1851885432992704:12345-0@lo:400:kworker/u8:3.0 Request processed in 40us (105us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100473.280207:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3052
00000100:00100000:0.0:1766100473.280234:0:3011:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acd0bfef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3011:1851885432992704:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100473.280255:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100473.280282:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992768
00000100:00100000:1.0:1766100473.280320:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1295
00000100:00100000:1.0:1766100473.280325:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432992768:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100473.280376:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000506576fd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432992768:12345-0@lo:400:kworker/u8:3.0 Request processed in 50us (121us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100473.280381:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1295
00010000:00080000:1.0:1766100473.280404:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@00000000286f9ae8 x1851885432992768/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100480 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:1.0:1766100473.280413:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000286f9ae8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432992768:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100473.280449:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100473.280493:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992640
00000100:00100000:1.0:1766100473.280520:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1916
00000100:00100000:1.0:1766100473.280523:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432992640:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100473.280616:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100473.280655:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c75ca47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432992640:12345-0@lo:8:kworker/u8:3.0 Request processed in 131us (206us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100473.280661:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1916
00000100:00080000:3.0:1766100473.280813:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1766100473.280817:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:3.0:1766100473.280821:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ec3fedd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992640:0@lo:8:kworker/u8:3.0
00010000:00010000:2.0:1766100473.855687:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100474.877819:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100475.901723:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100476.927042:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:1.0:1766100477.212402:0:3003:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100477.212554:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257472
00000100:00100000:0.0:1766100477.212860:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1290
00000100:00100000:0.0:1766100477.212868:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417257472:12345-192.168.203.52@tcp:17:kworker/1:0.0
00000100:00100000:0.0:1766100477.213268:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000286f9ae8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2228:x1851885417257472:12345-192.168.203.52@tcp:17:kworker/1:0.0 Request processed in 399us (873us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100477.213276:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1290
00000100:00100000:1.0:1766100477.725136:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:1.0:1766100477.725441:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:0.0:1766100477.725597:0:5513:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257600
00000100:00100000:0.0:1766100477.725678:0:5513:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 3053
00000100:00100000:0.0:1766100477.725687:0:5513:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417257600:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:0.0:1766100477.725821:0:5513:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ec3fedd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:56772462-1282-4783-8e1a-69be7ca0dd12+8:2229:x1851885417257600:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 134us (696us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100477.725831:0:5513:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 3053
00000100:00100000:3.0:1766100477.726596:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257536
00000100:00100000:3.0:1766100477.726736:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 2064
00000100:00100000:3.0:1766100477.726743:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417257536:12345-192.168.203.52@tcp:400:kworker/u8:3.0
00000100:00100000:3.0:1766100477.728016:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e2541a3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:d30215d4-f453-4d61-9211-cb0b369f1629+9:2228:x1851885417257536:12345-192.168.203.52@tcp:400:kworker/u8:3.0 Request processed in 1266us (2568us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100477.728030:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 2064
00000100:00100000:1.0:1766100477.728463:0:3001:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.52@tcp (source: 12345-192.168.203.52@tcp)
00000100:00100000:2.0:1766100477.729080:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885417257664
00000100:00100000:2.0:1766100477.729133:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.52@tcp, seq: 1917
00000100:00100000:2.0:1766100477.729147:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417257664:12345-192.168.203.52@tcp:8:kworker/u8:3.0
00010000:02020000:2.0:1766100477.729251:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.52@tcp (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:2.0:1766100477.729340:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acd0bfef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2227:x1851885417257664:12345-192.168.203.52@tcp:8:kworker/u8:3.0 Request processed in 197us (887us total) trans 0 rc -19/-19
00000100:00100000:2.0:1766100477.729348:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.52@tcp, seq: 1917
00010000:00010000:2.0:1766100477.951817:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1766100478.397762:0:38:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1766100478.397772:0:38:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery
00000100:00080000:1.0:1766100478.397777:0:38:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1766100478.397788:0:38:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4981
00000100:00080000:1.0:1766100478.397792:0:38:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s
00000100:00080000:1.0:1766100478.397801:0:38:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000c537a2f4 using connection 192.168.203.152@tcp/0@lo
00000100:00100000:1.0:1766100478.397832:0:38:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@00000000af55eb76 x1851885432992960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:''
00000100:00100000:3.0:1766100478.398555:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432992896:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100478.398649:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100478.398775:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432992832:0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100478.398816:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:3.0:1766100478.398871:0:5505:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992832
00000100:00100000:3.0:1766100478.398946:0:5505:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2065
00000100:00100000:3.0:1766100478.398955:0:5505:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432992832:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:3.0:1766100478.399029:0:5505:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006008c6d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:4d8e661b-5d90-4003-ae19-6056efb0a422+15:3009:x1851885432992832:12345-0@lo:400:kworker/u8:3.0 Request processed in 74us (213us total) trans 0 rc 0/0
00000100:00100000:3.0:1766100478.399038:0:5505:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2065
00000100:00100000:3.0:1766100478.399099:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6bb6a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:4d8e661b-5d90-4003-ae19-6056efb0a422:3009:1851885432992832:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100478.402205:0:3007:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af55eb76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992960:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100478.402293:0:3007:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100478.402366:0:3010:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2]
00000100:00100000:1.0:1766100478.402376:0:3010:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0edbad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432993024:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100478.402407:0:3010:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:1.0:1766100478.402448:0:6402:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992960
00000100:00100000:1.0:1766100478.402621:0:6402:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918
00000100:00100000:1.0:1766100478.402660:0:6402:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432992960:12345-0@lo:8:kworker/u8:3.0
00010000:02020000:1.0:1766100478.402891:0:6402:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server.
00000100:00100000:1.0:1766100478.402987:0:6402:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069d150bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3007:x1851885432992960:12345-0@lo:8:kworker/u8:3.0 Request processed in 327us (694us total) trans 0 rc -19/-19
00000100:00100000:1.0:1766100478.403007:0:6402:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918
00000100:00100000:0.0:1766100478.409038:0:3009:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1]
00000100:00100000:0.0:1766100478.409078:0:3009:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a733f90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432993088:0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100478.409183:0:3009:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo)
00000100:00100000:0.0:1766100478.409242:0:6301:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432993088
00000100:00100000:0.0:1766100478.409300:0:6301:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1296
00000100:00100000:0.0:1766100478.409308:0:6301:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432993088:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:0.0:1766100478.409382:0:6301:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e68d47de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3009:x1851885432993088:12345-0@lo:400:kworker/u8:3.0 Request processed in 73us (200us total) trans 0 rc 0/0
00000100:00100000:0.0:1766100478.409393:0:6301:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1296
00010000:00080000:0.0:1766100478.409441:0:3009:0:(ldlm_request.c:1487:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324)  req@000000002a733f90 x1851885432993088/t0(0) o400->lustre-OST0001-osc-MDT0000@0@lo:28/4 lens 224/224 e 0 to 0 dl 1766100485 ref 1 fl Rpc:RNQU/0/0 rc 0/0 job:'kworker/u8:3.0'
00000100:00100000:0.0:1766100478.409455:0:3009:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a733f90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3009:1851885432993088:0@lo:400:kworker/u8:3.0
00000100:00080000:0.0:1766100478.409553:0:3007:0:(import.c:86:import_set_state_nolock()) 00000000c537a2f4 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1766100478.409557:0:3007:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.152@tcp failed (-19)
00000100:00100000:0.0:1766100478.409562:0:3007:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af55eb76 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3007:1851885432992960:0@lo:8:kworker/u8:3.0
00000100:00100000:1.0:1766100478.409683:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432992896
00000100:00100000:1.0:1766100478.409730:0:5512:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1851885432993024
00000100:00100000:1.0:1766100478.409754:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3054
00000100:00100000:1.0:1766100478.409757:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000032a0d422 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432992896:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100478.409802:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000032a0d422 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3010:x1851885432992896:12345-0@lo:400:kworker/u8:3.0 Request processed in 43us (11157us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100478.409808:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3054
00000100:00100000:1.0:1766100478.409820:0:5512:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3055
00000100:00100000:1.0:1766100478.409823:0:5512:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432993024:12345-0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100478.409847:0:5512:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9ff3e80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3010:x1851885432993024:12345-0@lo:400:kworker/u8:3.0 Request processed in 23us (7440us total) trans 0 rc 0/0
00000100:00100000:1.0:1766100478.409853:0:5512:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3055
00000100:00100000:1.0:1766100478.409888:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0edbad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3010:1851885432993024:0@lo:400:kworker/u8:3.0
00000100:00100000:1.0:1766100478.409902:0:3010:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039f6e93a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3010:1851885432992896:0@lo:400:kworker/u8:3.0
00010000:00010000:1.0:1766100478.974636:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100480.000251:0:5501:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:2.0:1766100481.021693:0:7092:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00010000:1.0:1766100482.046688:0:5500:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
Debug log: 35491 lines, 35491 kept, 0 dropped, 0 bad.